From a991b64811f4d6adb6c7b31c0df52288eb06cf19 Mon Sep 17 00:00:00 2001 From: Sylvain Lebresne Date: Mon, 1 Sep 2014 18:54:46 +0200 Subject: [PATCH] Storage engine refactor, a.k.a CASSANDRA-8099 Initial patch, see ticket for details --- bin/sstablekeys | 56 - bin/sstablekeys.bat | 41 - build.xml | 2 +- guide_8099.md | 376 +++ .../cassandra/cache/AutoSavingCache.java | 2 +- .../cassandra/cache/CounterCacheKey.java | 29 +- .../apache/cassandra/cache/OHCProvider.java | 8 +- .../cache/SerializingCacheProvider.java | 9 +- .../apache/cassandra/config/CFMetaData.java | 1058 ++++--- .../cassandra/config/ColumnDefinition.java | 203 +- .../cassandra/config/DatabaseDescriptor.java | 8 +- .../config/YamlConfigurationLoader.java | 7 +- .../org/apache/cassandra/cql3/Attributes.java | 7 +- .../cassandra/cql3/ColumnCondition.java | 156 +- .../cassandra/cql3/ColumnIdentifier.java | 70 +- .../org/apache/cassandra/cql3/Constants.java | 31 +- src/java/org/apache/cassandra/cql3/Cql.g | 47 +- src/java/org/apache/cassandra/cql3/Lists.java | 113 +- src/java/org/apache/cassandra/cql3/Maps.java | 52 +- .../cassandra/cql3/MultiColumnRelation.java | 10 +- .../org/apache/cassandra/cql3/Operation.java | 14 +- .../org/apache/cassandra/cql3/Operator.java | 70 +- .../apache/cassandra/cql3/QueryProcessor.java | 69 +- src/java/org/apache/cassandra/cql3/Sets.java | 51 +- .../cassandra/cql3/SingleColumnRelation.java | 27 +- .../apache/cassandra/cql3/TokenRelation.java | 4 +- .../cassandra/cql3/UntypedResultSet.java | 45 +- .../cassandra/cql3/UpdateParameters.java | 160 +- .../cassandra/cql3/functions/TokenFct.java | 7 +- .../AbstractPrimaryKeyRestrictions.java | 11 +- .../restrictions/AbstractRestriction.java | 14 +- .../ForwardingPrimaryKeyRestrictions.java | 27 +- .../restrictions/MultiColumnRestriction.java | 70 +- .../PrimaryKeyRestrictionSet.java | 104 +- .../restrictions/PrimaryKeyRestrictions.java | 9 +- .../cql3/restrictions/Restriction.java | 32 +- .../cql3/restrictions/RestrictionSet.java | 12 +- .../cql3/restrictions/Restrictions.java | 15 +- .../ReversedPrimaryKeyRestrictions.java | 77 - .../restrictions/SingleColumnRestriction.java | 133 +- .../restrictions/StatementRestrictions.java | 146 +- .../cql3/restrictions/TokenFilter.java | 11 +- .../cql3/restrictions/TokenRestriction.java | 49 +- .../cassandra/cql3/selection/Selection.java | 57 +- .../cassandra/cql3/selection/Selector.java | 2 +- .../cql3/statements/AlterTableStatement.java | 80 +- .../cql3/statements/AlterTypeStatement.java | 41 +- .../cql3/statements/BatchStatement.java | 173 +- .../cql3/statements/CQL3CasRequest.java | 184 +- .../cql3/statements/CreateIndexStatement.java | 18 +- .../cql3/statements/CreateTableStatement.java | 300 +- .../cql3/statements/DeleteStatement.java | 65 +- .../cql3/statements/DropTypeStatement.java | 6 - .../statements/ModificationStatement.java | 340 ++- .../cql3/statements/SelectStatement.java | 719 ++--- .../cql3/statements/UpdateStatement.java | 126 +- .../org/apache/cassandra/db/AbstractCell.java | 236 -- .../db/AbstractClusteringPrefix.java | 93 + .../cassandra/db/AbstractLivenessInfo.java | 164 ++ .../cassandra/db/AbstractNativeCell.java | 710 ----- .../cassandra/db/AbstractRangeCommand.java | 101 - .../org/apache/cassandra/db/Aliasable.java | 62 + .../db/ArrayBackedSortedColumns.java | 774 ----- .../apache/cassandra/db/AtomDeserializer.java | 128 - .../cassandra/db/AtomicBTreeColumns.java | 578 ---- .../apache/cassandra/db/BatchlogManager.java | 21 +- .../org/apache/cassandra/db/BufferCell.java | 103 - .../cassandra/db/BufferCounterCell.java | 176 -- .../cassandra/db/BufferCounterUpdateCell.java | 96 - .../cassandra/db/BufferDeletedCell.java | 118 - .../cassandra/db/BufferExpiringCell.java | 187 -- .../org/apache/cassandra/db/CBuilder.java | 231 ++ .../org/apache/cassandra/db/CFRowAdder.java | 121 - src/java/org/apache/cassandra/db/Cell.java | 69 - .../IReadCommand.java => db/Clusterable.java} | 11 +- .../org/apache/cassandra/db/Clustering.java | 171 ++ .../cassandra/db/ClusteringComparator.java | 291 ++ .../apache/cassandra/db/ClusteringPrefix.java | 513 ++++ .../cassandra/db/CollationController.java | 334 --- .../org/apache/cassandra/db/ColumnFamily.java | 565 ---- .../cassandra/db/ColumnFamilySerializer.java | 172 -- .../cassandra/db/ColumnFamilyStore.java | 661 +---- .../org/apache/cassandra/db/ColumnIndex.java | 230 +- .../apache/cassandra/db/ColumnSerializer.java | 188 -- src/java/org/apache/cassandra/db/Columns.java | 535 ++++ .../apache/cassandra/db/CompactTables.java | 176 ++ .../org/apache/cassandra/db/Conflicts.java | 79 + .../org/apache/cassandra/db/CounterCell.java | 44 - .../apache/cassandra/db/CounterMutation.java | 225 +- .../org/apache/cassandra/db/DataRange.java | 488 ++-- .../org/apache/cassandra/db/DecoratedKey.java | 10 +- .../org/apache/cassandra/db/DeletedCell.java | 30 - .../org/apache/cassandra/db/DeletionInfo.java | 300 +- .../org/apache/cassandra/db/DeletionTime.java | 92 +- .../cassandra/db/DeletionTimeArray.java | 153 + .../org/apache/cassandra/db/ExpiringCell.java | 44 - .../cassandra/db/HintedHandOffManager.java | 265 +- .../org/apache/cassandra/db/IMutation.java | 7 +- .../apache/cassandra/db/IndexExpression.java | 121 - .../org/apache/cassandra/db/Keyspace.java | 61 +- .../org/apache/cassandra/db/LegacyLayout.java | 1301 +++++++++ .../org/apache/cassandra/db/LivenessInfo.java | 186 ++ .../cassandra/db/LivenessInfoArray.java | 174 ++ .../org/apache/cassandra/db/Memtable.java | 265 +- .../apache/cassandra/db/MultiCBuilder.java | 436 +++ .../org/apache/cassandra/db/Mutation.java | 229 +- .../org/apache/cassandra/db/NativeCell.java | 88 - .../cassandra/db/NativeCounterCell.java | 186 -- .../cassandra/db/NativeDeletedCell.java | 119 - .../cassandra/db/NativeExpiringCell.java | 190 -- .../org/apache/cassandra/db/OnDiskAtom.java | 102 - .../cassandra/db/PagedRangeCommand.java | 224 -- .../apache/cassandra/db/PartitionColumns.java | 184 ++ ...owPosition.java => PartitionPosition.java} | 13 +- .../db/PartitionRangeReadCommand.java | 288 ++ .../cassandra/db/RangeSliceCommand.java | 246 -- .../apache/cassandra/db/RangeSliceReply.java | 92 - .../apache/cassandra/db/RangeTombstone.java | 418 +-- .../cassandra/db/RangeTombstoneList.java | 596 ++-- .../org/apache/cassandra/db/ReadCommand.java | 524 +++- ...ndler.java => ReadCommandVerbHandler.java} | 28 +- .../apache/cassandra/db/ReadOrderGroup.java | 133 + .../org/apache/cassandra/db/ReadQuery.java | 118 + .../org/apache/cassandra/db/ReadResponse.java | 257 +- .../db/RetriedSliceFromReadCommand.java | 56 - .../cassandra/db/ReusableClustering.java | 82 + .../db/ReusableClusteringPrefix.java | 57 + .../cassandra/db/ReusableLivenessInfo.java | 65 + src/java/org/apache/cassandra/db/Row.java | 88 - .../apache/cassandra/db/RowIndexEntry.java | 35 +- .../cassandra/db/RowIteratorFactory.java | 172 -- .../apache/cassandra/db/RowUpdateBuilder.java | 316 ++ .../cassandra/db/SerializationHeader.java | 488 ++++ .../org/apache/cassandra/db/Serializers.java | 71 + .../apache/cassandra/db/SimpleClustering.java | 93 + .../cassandra/db/SimpleDeletionTime.java | 61 + .../cassandra/db/SimpleLivenessInfo.java | 75 + .../db/SinglePartitionNamesCommand.java | 249 ++ .../db/SinglePartitionReadCommand.java | 498 ++++ .../db/SinglePartitionSliceCommand.java | 232 ++ src/java/org/apache/cassandra/db/Slice.java | 652 +++++ .../cassandra/db/SliceByNamesReadCommand.java | 125 - .../cassandra/db/SliceFromReadCommand.java | 207 -- src/java/org/apache/cassandra/db/Slices.java | 898 ++++++ .../org/apache/cassandra/db/SuperColumns.java | 230 -- .../apache/cassandra/db/SystemKeyspace.java | 85 +- .../cassandra/db/UnfilteredDeserializer.java | 414 +++ ...yType.java => UnknownColumnException.java} | 35 +- .../AbstractSSTableIterator.java | 424 +++ .../IColumnIteratorFactory.java | 44 - .../db/columniterator/LazyColumnIterator.java | 100 - .../db/columniterator/OnDiskAtomIterator.java | 42 - .../db/columniterator/SSTableIterator.java | 292 ++ .../SSTableReversedIterator.java | 388 +++ .../db/commitlog/CommitLogArchiver.java | 2 +- .../db/commitlog/CommitLogDescriptor.java | 5 +- .../db/commitlog/CommitLogReplayer.java | 41 +- .../db/commitlog/CommitLogSegment.java | 8 +- .../db/compaction/AbstractCompactedRow.java | 65 - .../AbstractCompactionIterable.java | 83 - .../AbstractCompactionStrategy.java | 1 + .../db/compaction/CompactionController.java | 8 +- .../db/compaction/CompactionIterable.java | 96 - .../db/compaction/CompactionIterator.java | 299 ++ .../db/compaction/CompactionManager.java | 235 +- .../compaction/CompactionStrategyManager.java | 22 +- .../db/compaction/CompactionTask.java | 102 +- .../db/compaction/LazilyCompactedRow.java | 346 --- .../compaction/LeveledCompactionStrategy.java | 54 +- .../db/compaction/LeveledManifest.java | 19 +- .../cassandra/db/compaction/Scrubber.java | 118 +- .../cassandra/db/compaction/Upgrader.java | 21 +- .../cassandra/db/compaction/Verifier.java | 20 +- .../writers/CompactionAwareWriter.java | 12 +- .../writers/DefaultCompactionWriter.java | 12 +- .../writers/MajorLeveledCompactionWriter.java | 15 +- .../writers/MaxSSTableSizeWriter.java | 15 +- .../SplittingSizeTieredCompactionWriter.java | 15 +- .../db/composites/AbstractCType.java | 394 --- .../db/composites/AbstractCellNameType.java | 454 --- .../db/composites/AbstractComposite.java | 141 - .../AbstractCompoundCellNameType.java | 295 -- .../AbstractSimpleCellNameType.java | 210 -- .../db/composites/BoundedComposite.java | 104 - .../apache/cassandra/db/composites/CType.java | 141 - .../cassandra/db/composites/CellName.java | 78 - .../cassandra/db/composites/CellNameType.java | 215 -- .../cassandra/db/composites/CellNames.java | 109 - .../cassandra/db/composites/Composite.java | 78 - .../cassandra/db/composites/Composites.java | 150 - .../db/composites/CompositesBuilder.java | 313 -- .../db/composites/CompoundCType.java | 180 -- .../db/composites/CompoundComposite.java | 88 - .../db/composites/CompoundDenseCellName.java | 86 - .../composites/CompoundDenseCellNameType.java | 87 - .../db/composites/CompoundSparseCellName.java | 182 -- .../CompoundSparseCellNameType.java | 330 --- .../cassandra/db/composites/SimpleCType.java | 156 - .../db/composites/SimpleComposite.java | 79 - .../db/composites/SimpleDenseCellName.java | 83 - .../composites/SimpleDenseCellNameType.java | 80 - .../db/composites/SimpleSparseCellName.java | 104 - .../composites/SimpleSparseCellNameType.java | 100 - .../cassandra/db/context/CounterContext.java | 7 +- .../filter/AbstractClusteringIndexFilter.java | 110 + .../db/filter/ClusteringIndexFilter.java | 152 + .../db/filter/ClusteringIndexNamesFilter.java | 271 ++ .../db/filter/ClusteringIndexSliceFilter.java | 179 ++ .../cassandra/db/filter/ColumnCounter.java | 217 -- .../cassandra/db/filter/ColumnFilter.java | 437 +++ .../cassandra/db/filter/ColumnSlice.java | 289 -- .../db/filter/ColumnSubselection.java | 233 ++ .../cassandra/db/filter/DataLimits.java | 737 +++++ .../cassandra/db/filter/ExtendedFilter.java | 499 ---- .../cassandra/db/filter/IDiskAtomFilter.java | 146 - .../cassandra/db/filter/NamesQueryFilter.java | 301 -- .../cassandra/db/filter/QueryFilter.java | 262 -- .../apache/cassandra/db/filter/RowFilter.java | 784 +++++ .../cassandra/db/filter/SliceQueryFilter.java | 583 ---- .../TombstoneOverwhelmingException.java | 72 +- ...AbstractSimplePerColumnSecondaryIndex.java | 165 +- .../db/index/PerColumnSecondaryIndex.java | 70 +- .../db/index/PerRowSecondaryIndex.java | 26 +- .../cassandra/db/index/SecondaryIndex.java | 76 +- .../db/index/SecondaryIndexBuilder.java | 30 +- .../db/index/SecondaryIndexManager.java | 479 ++-- .../db/index/SecondaryIndexSearcher.java | 220 +- .../db/index/composites/CompositesIndex.java | 118 +- ...CompositesIndexIncludingCollectionKey.java | 51 +- .../CompositesIndexOnClusteringKey.java | 87 +- .../CompositesIndexOnCollectionKey.java | 15 +- ...ompositesIndexOnCollectionKeyAndValue.java | 52 +- .../CompositesIndexOnCollectionValue.java | 82 +- .../CompositesIndexOnPartitionKey.java | 60 +- .../composites/CompositesIndexOnRegular.java | 61 +- .../index/composites/CompositesSearcher.java | 448 ++- .../cassandra/db/index/keys/KeysIndex.java | 67 +- .../cassandra/db/index/keys/KeysSearcher.java | 299 +- .../db/lifecycle/SSTableIntervalTree.java | 14 +- .../apache/cassandra/db/lifecycle/View.java | 8 +- .../cassandra/db/marshal/AbstractType.java | 64 + .../cassandra/db/marshal/BooleanType.java | 6 + .../cassandra/db/marshal/CollectionType.java | 67 +- .../db/marshal/ColumnToCollectionType.java | 3 + .../cassandra/db/marshal/CompositeType.java | 25 +- .../db/marshal/CounterColumnType.java | 7 + .../apache/cassandra/db/marshal/DateType.java | 6 + .../cassandra/db/marshal/DoubleType.java | 6 + .../cassandra/db/marshal/EmptyType.java | 6 + .../cassandra/db/marshal/FloatType.java | 6 + .../cassandra/db/marshal/Int32Type.java | 5 + .../cassandra/db/marshal/LexicalUUIDType.java | 6 + .../apache/cassandra/db/marshal/ListType.java | 11 +- .../db/marshal/LocalByPartionerType.java | 12 +- .../apache/cassandra/db/marshal/LongType.java | 5 + .../apache/cassandra/db/marshal/MapType.java | 16 +- .../cassandra/db/marshal/ReversedType.java | 12 + .../apache/cassandra/db/marshal/SetType.java | 11 +- .../cassandra/db/marshal/TimeUUIDType.java | 6 + .../cassandra/db/marshal/TimestampType.java | 6 + .../apache/cassandra/db/marshal/UUIDType.java | 6 + .../db/partitions/AbstractPartitionData.java | 831 ++++++ .../AbstractUnfilteredPartitionIterator.java} | 20 +- .../ArrayBackedCachedPartition.java | 256 ++ .../db/partitions/ArrayBackedPartition.java | 104 + .../db/partitions/AtomicBTreePartition.java | 819 ++++++ .../db/partitions/CachedPartition.java | 96 + .../partitions/CountingPartitionIterator.java | 58 + .../CountingRowIterator.java} | 37 +- .../CountingUnfilteredPartitionIterator.java | 52 + .../CountingUnfilteredRowIterator.java | 64 + .../db/partitions/FilteredPartition.java | 142 + .../FilteringPartitionIterator.java | 146 + .../cassandra/db/partitions/Partition.java | 70 + .../partitions/PartitionIterator.java} | 32 +- .../db/partitions/PartitionIterators.java | 198 ++ .../db/partitions/PartitionUpdate.java | 764 +++++ .../SingletonUnfilteredPartitionIterator.java | 64 + .../TombstonePurgingPartitionIterator.java | 103 + .../UnfilteredPartitionIterator.java | 46 + .../UnfilteredPartitionIterators.java | 503 ++++ .../WrappingPartitionIterator.java} | 37 +- .../WrappingUnfilteredPartitionIterator.java | 120 + .../cassandra/db/rows/AbstractCell.java | 135 + .../db/rows/AbstractRangeTombstoneMarker.java | 71 + .../db/rows/AbstractReusableRow.java | 207 ++ .../apache/cassandra/db/rows/AbstractRow.java | 209 ++ .../rows/AbstractUnfilteredRowIterator.java | 107 + .../org/apache/cassandra/db/rows/Cell.java | 142 + .../apache/cassandra/db/rows/CellData.java | 275 ++ .../apache/cassandra/db/rows/CellPath.java | 127 + .../org/apache/cassandra/db/rows/Cells.java | 371 +++ .../apache/cassandra/db/rows/ColumnData.java | 61 + .../db/rows/ComplexRowDataBlock.java | 796 ++++++ .../CBuilder.java => rows/CounterCells.java} | 22 +- .../cassandra/db/rows/FilteringRow.java | 121 + .../db/rows/FilteringRowIterator.java | 126 + ...azilyInitializedUnfilteredRowIterator.java | 103 + .../cassandra/db/rows/MemtableRowData.java | 204 ++ .../db/rows/RangeTombstoneBoundMarker.java | 156 + .../db/rows/RangeTombstoneBoundaryMarker.java | 173 ++ .../db/rows/RangeTombstoneMarker.java | 283 ++ .../apache/cassandra/db/rows/ReusableRow.java | 104 + .../org/apache/cassandra/db/rows/Row.java | 555 ++++ .../db/rows/RowAndTombstoneMergeIterator.java | 170 ++ .../cassandra/db/rows/RowDataBlock.java | 275 ++ .../apache/cassandra/db/rows/RowIterator.java | 76 + .../cassandra/db/rows/RowIterators.java | 152 + .../apache/cassandra/db/rows/RowStats.java | 237 ++ .../org/apache/cassandra/db/rows/Rows.java | 205 ++ .../db/rows/SerializationHelper.java | 137 + .../cassandra/db/rows/SimpleRowDataBlock.java | 188 ++ .../rows/SliceableUnfilteredRowIterator.java | 39 + .../apache/cassandra/db/rows/StaticRow.java | 193 ++ .../TombstoneFilteringRow.java} | 31 +- .../apache/cassandra/db/rows/Unfiltered.java | 60 + .../db/rows/UnfilteredRowIterator.java | 102 + .../rows/UnfilteredRowIteratorSerializer.java | 306 ++ .../db/rows/UnfilteredRowIterators.java | 770 +++++ .../db/rows/UnfilteredSerializer.java | 706 +++++ .../apache/cassandra/db/rows/WrappingRow.java | 214 ++ .../db/rows/WrappingRowIterator.java | 79 + .../rows/WrappingUnfilteredRowIterator.java | 89 + .../apache/cassandra/dht/AbstractBounds.java | 9 +- src/java/org/apache/cassandra/dht/Bounds.java | 16 +- .../apache/cassandra/dht/ExcludingBounds.java | 10 + .../dht/IncludingExcludingBounds.java | 10 + src/java/org/apache/cassandra/dht/Range.java | 19 +- src/java/org/apache/cassandra/dht/Token.java | 10 +- .../sstable/AbstractSSTableSimpleWriter.java | 143 +- .../io/sstable/CQLSSTableWriter.java | 110 +- .../io/sstable/ColumnNameHelper.java | 241 -- .../cassandra/io/sstable/ColumnStats.java | 165 -- .../io/sstable/CorruptSSTableException.java | 2 +- .../cassandra/io/sstable/Descriptor.java | 3 +- .../cassandra/io/sstable/ISSTableScanner.java | 5 +- .../cassandra/io/sstable/IndexHelper.java | 132 +- .../cassandra/io/sstable/IndexSummary.java | 7 +- .../io/sstable/ReducingKeyIterator.java | 2 +- .../io/sstable/SSTableIdentityIterator.java | 143 +- .../cassandra/io/sstable/SSTableRewriter.java | 36 +- .../io/sstable/SSTableSimpleIterator.java | 176 ++ .../sstable/SSTableSimpleUnsortedWriter.java | 271 +- .../io/sstable/SSTableSimpleWriter.java | 91 +- .../io/sstable/format/SSTableFormat.java | 13 +- .../io/sstable/format/SSTableReader.java | 188 +- .../io/sstable/format/SSTableWriter.java | 84 +- .../cassandra/io/sstable/format/Version.java | 5 + .../io/sstable/format/big/BigFormat.java | 60 +- .../io/sstable/format/big/BigTableReader.java | 57 +- .../sstable/format/big/BigTableScanner.java | 193 +- .../io/sstable/format/big/BigTableWriter.java | 230 +- .../format/big/IndexedSliceReader.java | 542 ---- .../format/big/SSTableNamesIterator.java | 264 -- .../format/big/SSTableSliceIterator.java | 102 - .../sstable/format/big/SimpleSliceReader.java | 108 - .../metadata/LegacyMetadataSerializer.java | 20 +- .../sstable/metadata/MetadataCollector.java | 286 +- .../sstable/metadata/MetadataSerializer.java | 2 +- .../io/sstable/metadata/MetadataType.java | 6 +- .../io/sstable/metadata/StatsMetadata.java | 124 +- .../io/util/DataIntegrityMetadata.java | 11 +- .../apache/cassandra/io/util/FileUtils.java | 29 + .../cassandra/net/MessagingService.java | 15 +- .../apache/cassandra/repair/RepairJob.java | 6 +- .../apache/cassandra/repair/Validator.java | 32 +- .../cassandra/schema/LegacySchemaTables.java | 970 +++---- .../cassandra/serializers/ListSerializer.java | 4 +- .../cassandra/serializers/MapSerializer.java | 10 +- .../cassandra/serializers/SetSerializer.java | 6 +- .../service/AbstractReadExecutor.java | 58 +- .../service/AsyncRepairCallback.java | 4 +- .../apache/cassandra/service/CASRequest.java | 13 +- .../cassandra/service/CacheService.java | 74 +- .../cassandra/service/DataResolver.java | 428 +++ ...igestResolver.java => DigestResolver.java} | 75 +- .../cassandra/service/IResponseResolver.java | 43 - .../service/RangeSliceResponseResolver.java | 168 -- .../service/RangeSliceVerbHandler.java | 40 - .../cassandra/service/ReadCallback.java | 104 +- ...RowResolver.java => ResponseResolver.java} | 37 +- .../cassandra/service/RowDataResolver.java | 177 -- .../cassandra/service/StorageProxy.java | 1032 ++++--- .../cassandra/service/StorageService.java | 25 +- .../service/pager/AbstractQueryPager.java | 426 +-- .../service/pager/MultiPartitionPager.java | 146 +- .../service/pager/NamesQueryPager.java | 108 - .../cassandra/service/pager/PagingState.java | 20 +- .../cassandra/service/pager/QueryPager.java | 66 +- .../cassandra/service/pager/QueryPagers.java | 175 +- .../service/pager/RangeNamesQueryPager.java | 57 +- .../service/pager/RangeSliceQueryPager.java | 135 +- .../service/pager/SinglePartitionPager.java | 60 +- .../service/pager/SliceQueryPager.java | 118 - .../cassandra/service/paxos/Commit.java | 108 +- .../cassandra/service/paxos/PaxosState.java | 12 +- .../service/paxos/PrepareCallback.java | 3 +- .../service/paxos/PrepareResponse.java | 73 +- .../cassandra/streaming/StreamReader.java | 170 +- .../cassandra/streaming/StreamSession.java | 8 +- .../compress/CompressedStreamReader.java | 4 +- .../streaming/messages/FileMessageHeader.java | 32 +- .../messages/OutgoingFileMessage.java | 5 +- .../streaming/messages/StreamMessage.java | 3 +- .../cassandra/thrift/CassandraServer.java | 1174 +++++--- .../cassandra/thrift/ThriftConversion.java | 181 +- .../cassandra/thrift/ThriftResultsMerger.java | 317 +++ .../cassandra/thrift/ThriftValidation.java | 223 +- .../apache/cassandra/tools/SSTableExport.java | 480 ---- .../apache/cassandra/tools/SSTableImport.java | 568 ---- .../cassandra/tracing/TraceKeyspace.java | 47 +- .../transport/messages/QueryMessage.java | 2 +- .../apache/cassandra/triggers/ITrigger.java | 7 +- .../cassandra/triggers/TriggerExecutor.java | 100 +- .../cassandra/utils/ByteBufferUtil.java | 16 +- .../apache/cassandra/utils/FBUtilities.java | 40 +- .../apache/cassandra/utils/MergeIterator.java | 32 +- .../utils/NativeSSTableLoaderClient.java | 63 +- .../org/apache/cassandra/utils/Sorting.java | 254 ++ .../apache/cassandra/utils/btree/BTree.java | 86 +- .../utils/btree/BTreeSearchIterator.java | 71 +- .../cassandra/utils/btree/BTreeSet.java | 383 --- .../apache/cassandra/utils/btree/Builder.java | 8 +- .../apache/cassandra/utils/btree/Path.java | 17 +- .../cassandra/utils/btree/UpdateFunction.java | 39 +- .../utils/concurrent/Accumulator.java | 5 + .../cassandra/utils/memory/HeapPool.java | 102 +- .../utils/memory/MemtableAllocator.java | 21 +- .../utils/memory/MemtableBufferAllocator.java | 94 +- .../utils/memory/NativeAllocator.java | 33 +- .../apache/cassandra/utils/LongBTreeTest.java | 502 ---- test/conf/cassandra.yaml | 2 +- test/conf/logback-test.xml | 2 +- ...eyspace1-Standard3-jb-1-CompressionInfo.db | Bin 43 -> 0 bytes .../Keyspace1-Standard3-jb-1-Data.db | Bin 133 -> 0 bytes .../Keyspace1-Standard3-jb-1-Filter.db | Bin 24 -> 0 bytes .../Keyspace1-Standard3-jb-1-Index.db | Bin 90 -> 0 bytes .../Keyspace1-Standard3-jb-1-Summary.db | Bin 71 -> 0 bytes test/data/corrupt-sstables/la-1-big-CRC.db | Bin 0 -> 8 bytes test/data/corrupt-sstables/la-1-big-Data.db | Bin 0 -> 280 bytes .../corrupt-sstables/la-1-big-Digest.adler32 | 1 + test/data/corrupt-sstables/la-1-big-Filter.db | Bin 0 -> 24 bytes test/data/corrupt-sstables/la-1-big-Index.db | Bin 0 -> 105 bytes ...1-Statistics.db => la-1-big-Statistics.db} | Bin 4390 -> 4649 bytes .../data/corrupt-sstables/la-1-big-Summary.db | Bin 0 -> 83 bytes ...tandard3-jb-1-TOC.txt => la-1-big-TOC.txt} | 7 +- .../DropKeyspaceCommitLogRecycleTest.java | 91 - .../cassandra/db/LongFlushMemtableTest.java | 86 - .../apache/cassandra/db/LongKeyspaceTest.java | 82 - .../db/commitlog/ComitLogStress.java | 97 - .../db/commitlog/CommitLogStressTest.java | 41 +- .../db/compaction/LongCompactionsTest.java | 41 +- .../LongLeveledCompactionStrategyTest.java | 9 +- .../cassandra/AbstractReadCommandBuilder.java | 327 +++ .../org/apache/cassandra/EmbeddedServer.java | 83 - .../unit/org/apache/cassandra/MockSchema.java | 17 +- .../org/apache/cassandra/SchemaLoader.java | 437 +-- .../org/apache/cassandra/UpdateBuilder.java | 119 + test/unit/org/apache/cassandra/Util.java | 412 ++- .../cassandra/cache/AutoSavingCacheTest.java | 25 +- .../cassandra/cache/CacheProviderTest.java | 117 +- .../cassandra/config/CFMetaDataTest.java | 155 + .../config/ColumnDefinitionTest.java | 13 +- .../config/LegacySchemaTablesTest.java | 9 +- .../org/apache/cassandra/cql3/CQLTester.java | 140 +- .../cassandra/cql3/ColumnConditionTest.java | 97 +- .../org/apache/cassandra/cql3/DeleteTest.java | 9 +- .../cassandra/cql3/IndexQueryPagingTest.java | 88 + .../cql3/NonNativeTimestampTest.java | 50 +- .../cassandra/cql3/SimpleQueryTest.java | 532 ++++ .../cql3/ThriftCompatibilityTest.java | 17 +- .../PrimaryKeyRestrictionSetTest.java | 645 ++--- .../validation/entities/CollectionsTest.java | 2 +- .../validation/entities/CountersTest.java | 4 +- .../entities/FrozenCollectionsTest.java | 10 +- .../entities/SecondaryIndexTest.java | 53 +- .../cql3/validation/entities/UFAuthTest.java | 2 +- .../entities/UFIdentificationTest.java | 8 +- .../cql3/validation/entities/UFTest.java | 3 +- .../miscellaneous/CrcCheckChanceTest.java | 4 +- .../SSTableMetadataTrackingTest.java | 1 + .../cql3/validation/operations/AlterTest.java | 2 + .../validation/operations/CreateTest.java | 8 +- .../operations/SelectLimitTest.java | 10 +- .../SelectSingleColumnRelationTest.java | 2 +- .../validation/operations/SelectTest.java | 4 +- .../db/ArrayBackedSortedColumnsTest.java | 426 --- .../cassandra/db/BatchlogManagerTest.java | 64 +- .../org/apache/cassandra/db/CellTest.java | 124 +- .../org/apache/cassandra/db/CleanupTest.java | 82 +- .../cassandra/db/CollationControllerTest.java | 138 - .../cassandra/db/ColumnFamilyMetricTest.java | 37 +- .../cassandra/db/ColumnFamilyStoreTest.java | 2535 +++-------------- .../apache/cassandra/db/ColumnFamilyTest.java | 277 -- .../apache/cassandra/db/CommitLogTest.java | 349 ++- .../apache/cassandra/db/CounterCacheTest.java | 78 +- .../apache/cassandra/db/CounterCellTest.java | 437 +-- .../cassandra/db/CounterMutationTest.java | 240 +- .../cassandra/db/DeletePartitionTest.java | 97 + .../apache/cassandra/db/DirectoriesTest.java | 35 +- .../cassandra/db/HintedHandOffTest.java | 48 +- .../org/apache/cassandra/db/KeyCacheTest.java | 87 +- .../org/apache/cassandra/db/KeyspaceTest.java | 839 +++--- .../cassandra/db/MultiKeyspaceTest.java | 49 + .../apache/cassandra/db/MultitableTest.java | 80 - .../org/apache/cassandra/db/NameSortTest.java | 76 +- .../apache/cassandra/db/NativeCellTest.java | 251 -- .../cassandra/db/PartitionRangeReadTest.java | 375 +++ .../apache/cassandra/db/PartitionTest.java | 180 ++ .../cassandra/db/RangeTombstoneListTest.java | 348 +-- .../cassandra/db/RangeTombstoneTest.java | 489 ++-- .../apache/cassandra/db/ReadMessageTest.java | 178 +- ...t.java => RecoveryManagerFlushedTest.java} | 27 +- ... => RecoveryManagerMissingHeaderTest.java} | 31 +- .../cassandra/db/RecoveryManagerTest.java | 125 +- .../db/RecoveryManagerTruncateTest.java | 154 +- .../apache/cassandra/db/RemoveCellTest.java | 83 +- .../cassandra/db/RemoveColumnFamilyTest.java | 73 - .../db/RemoveColumnFamilyWithFlush1Test.java | 75 - .../db/RemoveColumnFamilyWithFlush2Test.java | 73 - .../cassandra/db/RemoveSubCellTest.java | 119 - .../apache/cassandra/db/RowCacheCQLTest.java | 10 +- .../org/apache/cassandra/db/RowCacheTest.java | 189 +- .../cassandra/db/RowIndexEntryTest.java | 71 +- .../apache/cassandra/db/RowIterationTest.java | 114 +- .../unit/org/apache/cassandra/db/RowTest.java | 199 +- .../org/apache/cassandra/db/ScrubTest.java | 260 +- .../cassandra/db/SecondaryIndexTest.java | 490 ++++ .../cassandra/db/SerializationsTest.java | 405 --- .../org/apache/cassandra/db/TimeSortTest.java | 135 +- .../org/apache/cassandra/db/VerifyTest.java | 141 +- .../db/commitlog/CommitLogTestReplayer.java | 4 +- .../db/commitlog/CommitLogUpgradeTest.java | 15 +- .../commitlog/CommitLogUpgradeTestMaker.java | 9 +- .../db/compaction/AntiCompactionTest.java | 140 +- .../BlacklistingCompactionsTest.java | 35 +- .../compaction/CompactionAwareWriterTest.java | 130 +- .../db/compaction/CompactionsPurgeTest.java | 287 +- .../db/compaction/CompactionsTest.java | 123 +- .../DateTieredCompactionStrategyTest.java | 38 +- .../LeveledCompactionStrategyTest.java | 52 +- .../db/compaction/OneCompactionTest.java | 67 +- .../SizeTieredCompactionStrategyTest.java | 29 +- .../db/compaction/TTLExpiryTest.java | 189 +- .../cassandra/db/composites/CTypeTest.java | 147 +- .../cassandra/db/filter/ColumnSliceTest.java | 495 ---- .../apache/cassandra/db/filter/SliceTest.java | 409 +++ .../db/index/PerRowSecondaryIndexTest.java | 186 +- .../cassandra/db/lifecycle/ViewTest.java | 6 +- .../db/marshal/CompositeTypeTest.java | 57 +- .../db/marshal/DynamicCompositeTypeTest.java | 79 +- .../cassandra/db/marshal/TypeCompareTest.java | 42 - .../cassandra/db/marshal/TypeParserTest.java | 12 +- .../db/marshal/TypeValidationTest.java | 26 - .../cassandra/dht/BootStrapperTest.java | 33 +- .../dht/ByteOrderedPartitionerTest.java | 33 +- .../cassandra/dht/KeyCollisionTest.java | 57 +- .../cassandra/dht/Murmur3PartitionerTest.java | 15 +- .../dht/OrderPreservingPartitionerTest.java | 33 +- .../cassandra/dht/PartitionerTestCase.java | 42 +- .../cassandra/dht/RandomPartitionerTest.java | 18 +- .../org/apache/cassandra/dht/RangeTest.java | 55 +- .../CompressedRandomAccessReaderTest.java | 13 +- .../CompressedSequentialWriterTest.java | 12 +- .../cassandra/io/compress/CompressorTest.java | 2 +- .../io/sstable/BigTableWriterTest.java | 24 +- .../sstable/CQLSSTableWriterClientTest.java | 9 +- .../io/sstable/CQLSSTableWriterTest.java | 27 +- .../cassandra/io/sstable/DescriptorTest.java | 5 +- .../cassandra/io/sstable/IndexHelperTest.java | 44 +- .../io/sstable/IndexSummaryManagerTest.java | 80 +- .../io/sstable/LegacySSTableTest.java | 92 +- .../io/sstable/SSTableLoaderTest.java | 34 +- .../io/sstable/SSTableMetadataTest.java | 182 +- .../io/sstable/SSTableReaderTest.java | 214 +- .../io/sstable/SSTableRewriterTest.java | 295 +- .../io/sstable/SSTableScannerTest.java | 141 +- .../io/sstable/SSTableSimpleWriterTest.java | 123 - .../cassandra/io/sstable/SSTableUtils.java | 63 +- .../metadata/MetadataSerializerTest.java | 34 +- .../locator/CloudstackSnitchTest.java | 3 +- .../locator/DynamicEndpointSnitchTest.java | 37 +- .../cassandra/locator/EC2SnitchTest.java | 18 +- .../locator/GoogleCloudSnitchTest.java | 18 +- .../GossipingPropertyFileSnitchTest.java | 4 +- .../locator/NetworkTopologyStrategyTest.java | 41 +- .../OldNetworkTopologyStrategyTest.java | 49 +- .../ReplicationStrategyEndpointCacheTest.java | 50 +- .../cassandra/locator/SimpleStrategyTest.java | 55 +- .../cassandra/locator/TokenMetadataTest.java | 42 +- .../cassandra/metrics/CQLMetricsTest.java | 10 +- .../cassandra/metrics/LatencyMetricsTest.java | 2 + .../cassandra/repair/LocalSyncTaskTest.java | 1 + .../cassandra/repair/RepairSessionTest.java | 1 + .../cassandra/repair/ValidatorTest.java | 41 +- .../repair/messages/RepairOptionTest.java | 5 +- .../org/apache/cassandra/schema/DefsTest.java | 346 ++- .../service/ActiveRepairServiceTest.java | 12 +- .../service/BatchlogEndpointFilterTest.java | 6 +- .../cassandra/service/DataResolverTest.java | 535 ++++ .../service/EmbeddedCassandraServiceTest.java | 12 +- .../service/LeaveAndBootstrapTest.java | 17 +- .../apache/cassandra/service/MoveTest.java | 16 +- .../cassandra/service/PaxosStateTest.java | 45 +- .../cassandra/service/QueryPagerTest.java | 282 +- .../cassandra/service/RowResolverTest.java | 160 -- .../cassandra/service/SerializationsTest.java | 1 + .../cassandra/service/StorageProxyTest.java | 41 +- .../service/StorageServiceServerTest.java | 9 +- .../service/pager/AbstractQueryPagerTest.java | 187 -- .../streaming/StreamTransferTaskTest.java | 8 +- .../streaming/StreamingTransferTest.java | 139 +- .../CompressedInputStreamTest.java | 14 +- .../cassandra/thrift/MultiSliceTest.java | 186 -- .../thrift/ThriftValidationTest.java | 205 -- .../cassandra/tools/SSTableExportTest.java | 405 --- .../cassandra/tools/SSTableImportTest.java | 278 -- .../transport/MessagePayloadTest.java | 2 - .../triggers/TriggerExecutorTest.java | 289 +- .../triggers/TriggersSchemaTest.java | 36 +- .../cassandra/triggers/TriggersTest.java | 75 +- .../org/apache/cassandra/utils/BTreeTest.java | 108 +- .../apache/cassandra/utils/BitSetTest.java | 6 +- .../cassandra/utils/BloomFilterTest.java | 52 +- .../cassandra/utils/ByteBufferUtilTest.java | 20 +- .../cassandra/utils/BytesReadTrackerTest.java | 6 +- .../cassandra/utils/EncodedStreamsTest.java | 75 +- .../utils/EstimatedHistogramTest.java | 2 +- .../cassandra/utils/FBUtilitiesTest.java | 5 +- .../org/apache/cassandra/utils/HexTest.java | 5 +- .../cassandra/utils/HistogramBuilderTest.java | 2 +- .../cassandra/utils/IntervalTreeTest.java | 131 +- .../utils/JVMStabilityInspectorTest.java | 11 +- .../apache/cassandra/utils/KeyGenerator.java | 18 +- .../cassandra/utils/MergeIteratorTest.java | 10 +- .../cassandra/utils/MerkleTreeTest.java | 7 +- .../cassandra/utils/SerializationsTest.java | 11 +- .../utils/StreamingHistogramTest.java | 8 +- .../cassandra/utils/TopKSamplerTest.java | 13 +- .../org/apache/cassandra/utils/UUIDTests.java | 8 +- .../utils/memory/NativeAllocatorTest.java | 1 - tools/bin/json2sstable | 51 - tools/bin/json2sstable.bat | 48 - tools/bin/sstable2json | 52 - tools/bin/sstable2json.bat | 48 - 645 files changed, 49618 insertions(+), 42464 deletions(-) delete mode 100755 bin/sstablekeys delete mode 100644 bin/sstablekeys.bat create mode 100644 guide_8099.md delete mode 100644 src/java/org/apache/cassandra/cql3/restrictions/ReversedPrimaryKeyRestrictions.java delete mode 100644 src/java/org/apache/cassandra/db/AbstractCell.java create mode 100644 src/java/org/apache/cassandra/db/AbstractClusteringPrefix.java create mode 100644 src/java/org/apache/cassandra/db/AbstractLivenessInfo.java delete mode 100644 src/java/org/apache/cassandra/db/AbstractNativeCell.java delete mode 100644 src/java/org/apache/cassandra/db/AbstractRangeCommand.java create mode 100644 src/java/org/apache/cassandra/db/Aliasable.java delete mode 100644 src/java/org/apache/cassandra/db/ArrayBackedSortedColumns.java delete mode 100644 src/java/org/apache/cassandra/db/AtomDeserializer.java delete mode 100644 src/java/org/apache/cassandra/db/AtomicBTreeColumns.java delete mode 100644 src/java/org/apache/cassandra/db/BufferCell.java delete mode 100644 src/java/org/apache/cassandra/db/BufferCounterCell.java delete mode 100644 src/java/org/apache/cassandra/db/BufferCounterUpdateCell.java delete mode 100644 src/java/org/apache/cassandra/db/BufferDeletedCell.java delete mode 100644 src/java/org/apache/cassandra/db/BufferExpiringCell.java create mode 100644 src/java/org/apache/cassandra/db/CBuilder.java delete mode 100644 src/java/org/apache/cassandra/db/CFRowAdder.java delete mode 100644 src/java/org/apache/cassandra/db/Cell.java rename src/java/org/apache/cassandra/{service/IReadCommand.java => db/Clusterable.java} (76%) create mode 100644 src/java/org/apache/cassandra/db/Clustering.java create mode 100644 src/java/org/apache/cassandra/db/ClusteringComparator.java create mode 100644 src/java/org/apache/cassandra/db/ClusteringPrefix.java delete mode 100644 src/java/org/apache/cassandra/db/CollationController.java delete mode 100644 src/java/org/apache/cassandra/db/ColumnFamily.java delete mode 100644 src/java/org/apache/cassandra/db/ColumnFamilySerializer.java delete mode 100644 src/java/org/apache/cassandra/db/ColumnSerializer.java create mode 100644 src/java/org/apache/cassandra/db/Columns.java create mode 100644 src/java/org/apache/cassandra/db/CompactTables.java create mode 100644 src/java/org/apache/cassandra/db/Conflicts.java delete mode 100644 src/java/org/apache/cassandra/db/CounterCell.java delete mode 100644 src/java/org/apache/cassandra/db/DeletedCell.java create mode 100644 src/java/org/apache/cassandra/db/DeletionTimeArray.java delete mode 100644 src/java/org/apache/cassandra/db/ExpiringCell.java delete mode 100644 src/java/org/apache/cassandra/db/IndexExpression.java create mode 100644 src/java/org/apache/cassandra/db/LegacyLayout.java create mode 100644 src/java/org/apache/cassandra/db/LivenessInfo.java create mode 100644 src/java/org/apache/cassandra/db/LivenessInfoArray.java create mode 100644 src/java/org/apache/cassandra/db/MultiCBuilder.java delete mode 100644 src/java/org/apache/cassandra/db/NativeCell.java delete mode 100644 src/java/org/apache/cassandra/db/NativeCounterCell.java delete mode 100644 src/java/org/apache/cassandra/db/NativeDeletedCell.java delete mode 100644 src/java/org/apache/cassandra/db/NativeExpiringCell.java delete mode 100644 src/java/org/apache/cassandra/db/OnDiskAtom.java delete mode 100644 src/java/org/apache/cassandra/db/PagedRangeCommand.java create mode 100644 src/java/org/apache/cassandra/db/PartitionColumns.java rename src/java/org/apache/cassandra/db/{RowPosition.java => PartitionPosition.java} (88%) create mode 100644 src/java/org/apache/cassandra/db/PartitionRangeReadCommand.java delete mode 100644 src/java/org/apache/cassandra/db/RangeSliceCommand.java delete mode 100644 src/java/org/apache/cassandra/db/RangeSliceReply.java rename src/java/org/apache/cassandra/db/{ReadVerbHandler.java => ReadCommandVerbHandler.java} (68%) create mode 100644 src/java/org/apache/cassandra/db/ReadOrderGroup.java create mode 100644 src/java/org/apache/cassandra/db/ReadQuery.java delete mode 100644 src/java/org/apache/cassandra/db/RetriedSliceFromReadCommand.java create mode 100644 src/java/org/apache/cassandra/db/ReusableClustering.java create mode 100644 src/java/org/apache/cassandra/db/ReusableClusteringPrefix.java create mode 100644 src/java/org/apache/cassandra/db/ReusableLivenessInfo.java delete mode 100644 src/java/org/apache/cassandra/db/Row.java delete mode 100644 src/java/org/apache/cassandra/db/RowIteratorFactory.java create mode 100644 src/java/org/apache/cassandra/db/RowUpdateBuilder.java create mode 100644 src/java/org/apache/cassandra/db/SerializationHeader.java create mode 100644 src/java/org/apache/cassandra/db/Serializers.java create mode 100644 src/java/org/apache/cassandra/db/SimpleClustering.java create mode 100644 src/java/org/apache/cassandra/db/SimpleDeletionTime.java create mode 100644 src/java/org/apache/cassandra/db/SimpleLivenessInfo.java create mode 100644 src/java/org/apache/cassandra/db/SinglePartitionNamesCommand.java create mode 100644 src/java/org/apache/cassandra/db/SinglePartitionReadCommand.java create mode 100644 src/java/org/apache/cassandra/db/SinglePartitionSliceCommand.java create mode 100644 src/java/org/apache/cassandra/db/Slice.java delete mode 100644 src/java/org/apache/cassandra/db/SliceByNamesReadCommand.java delete mode 100644 src/java/org/apache/cassandra/db/SliceFromReadCommand.java create mode 100644 src/java/org/apache/cassandra/db/Slices.java delete mode 100644 src/java/org/apache/cassandra/db/SuperColumns.java create mode 100644 src/java/org/apache/cassandra/db/UnfilteredDeserializer.java rename src/java/org/apache/cassandra/db/{ColumnFamilyType.java => UnknownColumnException.java} (51%) create mode 100644 src/java/org/apache/cassandra/db/columniterator/AbstractSSTableIterator.java delete mode 100644 src/java/org/apache/cassandra/db/columniterator/IColumnIteratorFactory.java delete mode 100644 src/java/org/apache/cassandra/db/columniterator/LazyColumnIterator.java delete mode 100644 src/java/org/apache/cassandra/db/columniterator/OnDiskAtomIterator.java create mode 100644 src/java/org/apache/cassandra/db/columniterator/SSTableIterator.java create mode 100644 src/java/org/apache/cassandra/db/columniterator/SSTableReversedIterator.java delete mode 100644 src/java/org/apache/cassandra/db/compaction/AbstractCompactedRow.java delete mode 100644 src/java/org/apache/cassandra/db/compaction/AbstractCompactionIterable.java delete mode 100644 src/java/org/apache/cassandra/db/compaction/CompactionIterable.java create mode 100644 src/java/org/apache/cassandra/db/compaction/CompactionIterator.java delete mode 100644 src/java/org/apache/cassandra/db/compaction/LazilyCompactedRow.java delete mode 100644 src/java/org/apache/cassandra/db/composites/AbstractCType.java delete mode 100644 src/java/org/apache/cassandra/db/composites/AbstractCellNameType.java delete mode 100644 src/java/org/apache/cassandra/db/composites/AbstractComposite.java delete mode 100644 src/java/org/apache/cassandra/db/composites/AbstractCompoundCellNameType.java delete mode 100644 src/java/org/apache/cassandra/db/composites/AbstractSimpleCellNameType.java delete mode 100644 src/java/org/apache/cassandra/db/composites/BoundedComposite.java delete mode 100644 src/java/org/apache/cassandra/db/composites/CType.java delete mode 100644 src/java/org/apache/cassandra/db/composites/CellName.java delete mode 100644 src/java/org/apache/cassandra/db/composites/CellNameType.java delete mode 100644 src/java/org/apache/cassandra/db/composites/CellNames.java delete mode 100644 src/java/org/apache/cassandra/db/composites/Composite.java delete mode 100644 src/java/org/apache/cassandra/db/composites/Composites.java delete mode 100644 src/java/org/apache/cassandra/db/composites/CompositesBuilder.java delete mode 100644 src/java/org/apache/cassandra/db/composites/CompoundCType.java delete mode 100644 src/java/org/apache/cassandra/db/composites/CompoundComposite.java delete mode 100644 src/java/org/apache/cassandra/db/composites/CompoundDenseCellName.java delete mode 100644 src/java/org/apache/cassandra/db/composites/CompoundDenseCellNameType.java delete mode 100644 src/java/org/apache/cassandra/db/composites/CompoundSparseCellName.java delete mode 100644 src/java/org/apache/cassandra/db/composites/CompoundSparseCellNameType.java delete mode 100644 src/java/org/apache/cassandra/db/composites/SimpleCType.java delete mode 100644 src/java/org/apache/cassandra/db/composites/SimpleComposite.java delete mode 100644 src/java/org/apache/cassandra/db/composites/SimpleDenseCellName.java delete mode 100644 src/java/org/apache/cassandra/db/composites/SimpleDenseCellNameType.java delete mode 100644 src/java/org/apache/cassandra/db/composites/SimpleSparseCellName.java delete mode 100644 src/java/org/apache/cassandra/db/composites/SimpleSparseCellNameType.java create mode 100644 src/java/org/apache/cassandra/db/filter/AbstractClusteringIndexFilter.java create mode 100644 src/java/org/apache/cassandra/db/filter/ClusteringIndexFilter.java create mode 100644 src/java/org/apache/cassandra/db/filter/ClusteringIndexNamesFilter.java create mode 100644 src/java/org/apache/cassandra/db/filter/ClusteringIndexSliceFilter.java delete mode 100644 src/java/org/apache/cassandra/db/filter/ColumnCounter.java create mode 100644 src/java/org/apache/cassandra/db/filter/ColumnFilter.java delete mode 100644 src/java/org/apache/cassandra/db/filter/ColumnSlice.java create mode 100644 src/java/org/apache/cassandra/db/filter/ColumnSubselection.java create mode 100644 src/java/org/apache/cassandra/db/filter/DataLimits.java delete mode 100644 src/java/org/apache/cassandra/db/filter/ExtendedFilter.java delete mode 100644 src/java/org/apache/cassandra/db/filter/IDiskAtomFilter.java delete mode 100644 src/java/org/apache/cassandra/db/filter/NamesQueryFilter.java delete mode 100644 src/java/org/apache/cassandra/db/filter/QueryFilter.java create mode 100644 src/java/org/apache/cassandra/db/filter/RowFilter.java delete mode 100644 src/java/org/apache/cassandra/db/filter/SliceQueryFilter.java create mode 100644 src/java/org/apache/cassandra/db/partitions/AbstractPartitionData.java rename src/java/org/apache/cassandra/db/{CounterUpdateCell.java => partitions/AbstractUnfilteredPartitionIterator.java} (65%) create mode 100644 src/java/org/apache/cassandra/db/partitions/ArrayBackedCachedPartition.java create mode 100644 src/java/org/apache/cassandra/db/partitions/ArrayBackedPartition.java create mode 100644 src/java/org/apache/cassandra/db/partitions/AtomicBTreePartition.java create mode 100644 src/java/org/apache/cassandra/db/partitions/CachedPartition.java create mode 100644 src/java/org/apache/cassandra/db/partitions/CountingPartitionIterator.java rename src/java/org/apache/cassandra/db/{composites/SimpleSparseInternedCellName.java => partitions/CountingRowIterator.java} (54%) create mode 100644 src/java/org/apache/cassandra/db/partitions/CountingUnfilteredPartitionIterator.java create mode 100644 src/java/org/apache/cassandra/db/partitions/CountingUnfilteredRowIterator.java create mode 100644 src/java/org/apache/cassandra/db/partitions/FilteredPartition.java create mode 100644 src/java/org/apache/cassandra/db/partitions/FilteringPartitionIterator.java create mode 100644 src/java/org/apache/cassandra/db/partitions/Partition.java rename src/java/org/apache/cassandra/{service/pager/Pageable.java => db/partitions/PartitionIterator.java} (51%) create mode 100644 src/java/org/apache/cassandra/db/partitions/PartitionIterators.java create mode 100644 src/java/org/apache/cassandra/db/partitions/PartitionUpdate.java create mode 100644 src/java/org/apache/cassandra/db/partitions/SingletonUnfilteredPartitionIterator.java create mode 100644 src/java/org/apache/cassandra/db/partitions/TombstonePurgingPartitionIterator.java create mode 100644 src/java/org/apache/cassandra/db/partitions/UnfilteredPartitionIterator.java create mode 100644 src/java/org/apache/cassandra/db/partitions/UnfilteredPartitionIterators.java rename src/java/org/apache/cassandra/{cql3/CQL3Row.java => db/partitions/WrappingPartitionIterator.java} (59%) create mode 100644 src/java/org/apache/cassandra/db/partitions/WrappingUnfilteredPartitionIterator.java create mode 100644 src/java/org/apache/cassandra/db/rows/AbstractCell.java create mode 100644 src/java/org/apache/cassandra/db/rows/AbstractRangeTombstoneMarker.java create mode 100644 src/java/org/apache/cassandra/db/rows/AbstractReusableRow.java create mode 100644 src/java/org/apache/cassandra/db/rows/AbstractRow.java create mode 100644 src/java/org/apache/cassandra/db/rows/AbstractUnfilteredRowIterator.java create mode 100644 src/java/org/apache/cassandra/db/rows/Cell.java create mode 100644 src/java/org/apache/cassandra/db/rows/CellData.java create mode 100644 src/java/org/apache/cassandra/db/rows/CellPath.java create mode 100644 src/java/org/apache/cassandra/db/rows/Cells.java create mode 100644 src/java/org/apache/cassandra/db/rows/ColumnData.java create mode 100644 src/java/org/apache/cassandra/db/rows/ComplexRowDataBlock.java rename src/java/org/apache/cassandra/db/{composites/CBuilder.java => rows/CounterCells.java} (66%) create mode 100644 src/java/org/apache/cassandra/db/rows/FilteringRow.java create mode 100644 src/java/org/apache/cassandra/db/rows/FilteringRowIterator.java create mode 100644 src/java/org/apache/cassandra/db/rows/LazilyInitializedUnfilteredRowIterator.java create mode 100644 src/java/org/apache/cassandra/db/rows/MemtableRowData.java create mode 100644 src/java/org/apache/cassandra/db/rows/RangeTombstoneBoundMarker.java create mode 100644 src/java/org/apache/cassandra/db/rows/RangeTombstoneBoundaryMarker.java create mode 100644 src/java/org/apache/cassandra/db/rows/RangeTombstoneMarker.java create mode 100644 src/java/org/apache/cassandra/db/rows/ReusableRow.java create mode 100644 src/java/org/apache/cassandra/db/rows/Row.java create mode 100644 src/java/org/apache/cassandra/db/rows/RowAndTombstoneMergeIterator.java create mode 100644 src/java/org/apache/cassandra/db/rows/RowDataBlock.java create mode 100644 src/java/org/apache/cassandra/db/rows/RowIterator.java create mode 100644 src/java/org/apache/cassandra/db/rows/RowIterators.java create mode 100644 src/java/org/apache/cassandra/db/rows/RowStats.java create mode 100644 src/java/org/apache/cassandra/db/rows/Rows.java create mode 100644 src/java/org/apache/cassandra/db/rows/SerializationHelper.java create mode 100644 src/java/org/apache/cassandra/db/rows/SimpleRowDataBlock.java create mode 100644 src/java/org/apache/cassandra/db/rows/SliceableUnfilteredRowIterator.java create mode 100644 src/java/org/apache/cassandra/db/rows/StaticRow.java rename src/java/org/apache/cassandra/db/{columniterator/IdentityQueryFilter.java => rows/TombstoneFilteringRow.java} (60%) create mode 100644 src/java/org/apache/cassandra/db/rows/Unfiltered.java create mode 100644 src/java/org/apache/cassandra/db/rows/UnfilteredRowIterator.java create mode 100644 src/java/org/apache/cassandra/db/rows/UnfilteredRowIteratorSerializer.java create mode 100644 src/java/org/apache/cassandra/db/rows/UnfilteredRowIterators.java create mode 100644 src/java/org/apache/cassandra/db/rows/UnfilteredSerializer.java create mode 100644 src/java/org/apache/cassandra/db/rows/WrappingRow.java create mode 100644 src/java/org/apache/cassandra/db/rows/WrappingRowIterator.java create mode 100644 src/java/org/apache/cassandra/db/rows/WrappingUnfilteredRowIterator.java delete mode 100644 src/java/org/apache/cassandra/io/sstable/ColumnNameHelper.java delete mode 100644 src/java/org/apache/cassandra/io/sstable/ColumnStats.java create mode 100644 src/java/org/apache/cassandra/io/sstable/SSTableSimpleIterator.java delete mode 100644 src/java/org/apache/cassandra/io/sstable/format/big/IndexedSliceReader.java delete mode 100644 src/java/org/apache/cassandra/io/sstable/format/big/SSTableNamesIterator.java delete mode 100644 src/java/org/apache/cassandra/io/sstable/format/big/SSTableSliceIterator.java delete mode 100644 src/java/org/apache/cassandra/io/sstable/format/big/SimpleSliceReader.java create mode 100644 src/java/org/apache/cassandra/service/DataResolver.java rename src/java/org/apache/cassandra/service/{RowDigestResolver.java => DigestResolver.java} (54%) delete mode 100644 src/java/org/apache/cassandra/service/IResponseResolver.java delete mode 100644 src/java/org/apache/cassandra/service/RangeSliceResponseResolver.java delete mode 100644 src/java/org/apache/cassandra/service/RangeSliceVerbHandler.java rename src/java/org/apache/cassandra/service/{AbstractRowResolver.java => ResponseResolver.java} (60%) delete mode 100644 src/java/org/apache/cassandra/service/RowDataResolver.java delete mode 100644 src/java/org/apache/cassandra/service/pager/NamesQueryPager.java delete mode 100644 src/java/org/apache/cassandra/service/pager/SliceQueryPager.java create mode 100644 src/java/org/apache/cassandra/thrift/ThriftResultsMerger.java delete mode 100644 src/java/org/apache/cassandra/tools/SSTableExport.java delete mode 100644 src/java/org/apache/cassandra/tools/SSTableImport.java create mode 100644 src/java/org/apache/cassandra/utils/Sorting.java delete mode 100644 src/java/org/apache/cassandra/utils/btree/BTreeSet.java delete mode 100644 test/burn/org/apache/cassandra/utils/LongBTreeTest.java delete mode 100644 test/data/corrupt-sstables/Keyspace1-Standard3-jb-1-CompressionInfo.db delete mode 100644 test/data/corrupt-sstables/Keyspace1-Standard3-jb-1-Data.db delete mode 100644 test/data/corrupt-sstables/Keyspace1-Standard3-jb-1-Filter.db delete mode 100644 test/data/corrupt-sstables/Keyspace1-Standard3-jb-1-Index.db delete mode 100644 test/data/corrupt-sstables/Keyspace1-Standard3-jb-1-Summary.db create mode 100644 test/data/corrupt-sstables/la-1-big-CRC.db create mode 100644 test/data/corrupt-sstables/la-1-big-Data.db create mode 100644 test/data/corrupt-sstables/la-1-big-Digest.adler32 create mode 100644 test/data/corrupt-sstables/la-1-big-Filter.db create mode 100644 test/data/corrupt-sstables/la-1-big-Index.db rename test/data/corrupt-sstables/{Keyspace1-Standard3-jb-1-Statistics.db => la-1-big-Statistics.db} (83%) create mode 100644 test/data/corrupt-sstables/la-1-big-Summary.db rename test/data/corrupt-sstables/{Keyspace1-Standard3-jb-1-TOC.txt => la-1-big-TOC.txt} (73%) delete mode 100644 test/long/org/apache/cassandra/cql3/DropKeyspaceCommitLogRecycleTest.java delete mode 100644 test/long/org/apache/cassandra/db/LongFlushMemtableTest.java delete mode 100644 test/long/org/apache/cassandra/db/LongKeyspaceTest.java delete mode 100644 test/long/org/apache/cassandra/db/commitlog/ComitLogStress.java create mode 100644 test/unit/org/apache/cassandra/AbstractReadCommandBuilder.java delete mode 100644 test/unit/org/apache/cassandra/EmbeddedServer.java create mode 100644 test/unit/org/apache/cassandra/UpdateBuilder.java create mode 100644 test/unit/org/apache/cassandra/config/CFMetaDataTest.java create mode 100644 test/unit/org/apache/cassandra/cql3/IndexQueryPagingTest.java create mode 100644 test/unit/org/apache/cassandra/cql3/SimpleQueryTest.java delete mode 100644 test/unit/org/apache/cassandra/db/ArrayBackedSortedColumnsTest.java delete mode 100644 test/unit/org/apache/cassandra/db/CollationControllerTest.java delete mode 100644 test/unit/org/apache/cassandra/db/ColumnFamilyTest.java create mode 100644 test/unit/org/apache/cassandra/db/DeletePartitionTest.java create mode 100644 test/unit/org/apache/cassandra/db/MultiKeyspaceTest.java delete mode 100644 test/unit/org/apache/cassandra/db/MultitableTest.java delete mode 100644 test/unit/org/apache/cassandra/db/NativeCellTest.java create mode 100644 test/unit/org/apache/cassandra/db/PartitionRangeReadTest.java create mode 100644 test/unit/org/apache/cassandra/db/PartitionTest.java rename test/unit/org/apache/cassandra/db/{RecoveryManager2Test.java => RecoveryManagerFlushedTest.java} (79%) rename test/unit/org/apache/cassandra/db/{RecoveryManager3Test.java => RecoveryManagerMissingHeaderTest.java} (71%) delete mode 100644 test/unit/org/apache/cassandra/db/RemoveColumnFamilyTest.java delete mode 100644 test/unit/org/apache/cassandra/db/RemoveColumnFamilyWithFlush1Test.java delete mode 100644 test/unit/org/apache/cassandra/db/RemoveColumnFamilyWithFlush2Test.java delete mode 100644 test/unit/org/apache/cassandra/db/RemoveSubCellTest.java create mode 100644 test/unit/org/apache/cassandra/db/SecondaryIndexTest.java delete mode 100644 test/unit/org/apache/cassandra/db/SerializationsTest.java delete mode 100644 test/unit/org/apache/cassandra/db/filter/ColumnSliceTest.java create mode 100644 test/unit/org/apache/cassandra/db/filter/SliceTest.java delete mode 100644 test/unit/org/apache/cassandra/io/sstable/SSTableSimpleWriterTest.java create mode 100644 test/unit/org/apache/cassandra/service/DataResolverTest.java delete mode 100644 test/unit/org/apache/cassandra/service/RowResolverTest.java delete mode 100644 test/unit/org/apache/cassandra/service/pager/AbstractQueryPagerTest.java rename test/unit/org/apache/cassandra/streaming/{compress => compression}/CompressedInputStreamTest.java (93%) delete mode 100644 test/unit/org/apache/cassandra/thrift/MultiSliceTest.java delete mode 100644 test/unit/org/apache/cassandra/thrift/ThriftValidationTest.java delete mode 100644 test/unit/org/apache/cassandra/tools/SSTableExportTest.java delete mode 100644 test/unit/org/apache/cassandra/tools/SSTableImportTest.java delete mode 100755 tools/bin/json2sstable delete mode 100644 tools/bin/json2sstable.bat delete mode 100755 tools/bin/sstable2json delete mode 100644 tools/bin/sstable2json.bat diff --git a/bin/sstablekeys b/bin/sstablekeys deleted file mode 100755 index 77d2e64c89..0000000000 --- a/bin/sstablekeys +++ /dev/null @@ -1,56 +0,0 @@ -#!/bin/sh - -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -if [ "x$CASSANDRA_INCLUDE" = "x" ]; then - for include in "`dirname "$0"`/cassandra.in.sh" \ - "$HOME/.cassandra.in.sh" \ - /usr/share/cassandra/cassandra.in.sh \ - /usr/local/share/cassandra/cassandra.in.sh \ - /opt/cassandra/cassandra.in.sh; do - if [ -r "$include" ]; then - . "$include" - break - fi - done -elif [ -r "$CASSANDRA_INCLUDE" ]; then - . "$CASSANDRA_INCLUDE" -fi - - -# Use JAVA_HOME if set, otherwise look for java in PATH -if [ -x "$JAVA_HOME/bin/java" ]; then - JAVA="$JAVA_HOME/bin/java" -else - JAVA="`which java`" -fi - -if [ -z "$CLASSPATH" ]; then - echo "You must set the CLASSPATH var" >&2 - exit 1 -fi -if [ $# -eq "0" ]; then - echo "Usage: `basename "$0"` " - exit 2 -fi - -"$JAVA" $JAVA_AGENT -cp "$CLASSPATH" $JVM_OPTS -Dstorage-config="$CASSANDRA_CONF" \ - -Dcassandra.storagedir="$cassandra_storagedir" \ - -Dlogback.configurationFile=logback-tools.xml \ - org.apache.cassandra.tools.SSTableExport "$@" -e - -# vi:ai sw=4 ts=4 tw=0 et diff --git a/bin/sstablekeys.bat b/bin/sstablekeys.bat deleted file mode 100644 index 0d0cf95eb0..0000000000 --- a/bin/sstablekeys.bat +++ /dev/null @@ -1,41 +0,0 @@ -@REM -@REM Licensed to the Apache Software Foundation (ASF) under one or more -@REM contributor license agreements. See the NOTICE file distributed with -@REM this work for additional information regarding copyright ownership. -@REM The ASF licenses this file to You under the Apache License, Version 2.0 -@REM (the "License"); you may not use this file except in compliance with -@REM the License. You may obtain a copy of the License at -@REM -@REM http://www.apache.org/licenses/LICENSE-2.0 -@REM -@REM Unless required by applicable law or agreed to in writing, software -@REM distributed under the License is distributed on an "AS IS" BASIS, -@REM WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -@REM See the License for the specific language governing permissions and -@REM limitations under the License. - -@echo off -if "%OS%" == "Windows_NT" setlocal - -pushd "%~dp0" -call cassandra.in.bat - -if NOT DEFINED CASSANDRA_MAIN set CASSANDRA_MAIN=org.apache.cassandra.tools.SSTableExport -if NOT DEFINED JAVA_HOME goto :err - -REM ***** JAVA options ***** -set JAVA_OPTS=^ - -Dlogback.configurationFile=logback-tools.xml - -set TOOLS_PARAMS= - -"%JAVA_HOME%\bin\java" %JAVA_OPTS% %CASSANDRA_PARAMS% -cp %CASSANDRA_CLASSPATH% "%CASSANDRA_MAIN%" %1 -e -goto finally - -:err -echo JAVA_HOME environment variable must be set! -pause - -:finally - -ENDLOCAL diff --git a/build.xml b/build.xml index a2b3e74ac7..7c05d7fec3 100644 --- a/build.xml +++ b/build.xml @@ -94,7 +94,7 @@ - + diff --git a/guide_8099.md b/guide_8099.md new file mode 100644 index 0000000000..627b9dca0c --- /dev/null +++ b/guide_8099.md @@ -0,0 +1,376 @@ +Overview of CASSANDRA-8099 changes +================================== + +The goal of this document is to provide an overview of the main changes done in +CASSANDRA-8099 so it's easier to dive into the patch. This assumes knowledge of +the pre-existing code. + +CASSANDRA-8099 refactors the abstractions used by the storage engine and as +such impact most of the code of said engine. The changes can be though of as +following two main guidelines: +1. the new abstractions are much more iterator-based (i.e. it tries harder to + avoid materializing everything in memory), +2. and they are closer to the CQL representation of things (i.e. the storage + engine is aware of more structure, making it able to optimize accordingly). +Note that while those changes have heavy impact on the actual code, the basic +mechanisms of the read and write paths are largely unchanged. + +In the following, I'll start by describe the new abstractions introduced by the +patch. I'll then provide a quick reference of existing class to what it becomes +in the patch, after which I'll discuss how the refactor handles a number of +more specific points. Lastly, the patch introduces some change to the on-wire +and on-disk format so I'll discuss those quickly. + + +Main new abstractions +--------------------- + +### Atom: Row and RangeTombstoneMarker + +Where the existing storage engine is mainly handling cells, the new engine +groups cells into rows, and rows becomes the more central building block. A +`Row` is identified by the value of it's clustering columns which are stored in +a `Clustering` object (see below), and it associate a number of cells to each +of its non-PK non-static columns (we'll discuss static columns more +specifically later). + +The patch distinguishes 2 kind of columns: simple and complex ones. The +_simple_ columns can have only 1 cell associated to them (or none), while the +_complex_ ones will have an arbitrary number of cells associated. Currently, +the complex columns are only the non frozen collections (but we'll have +non-frozen udt at some point and who knows what in the future). + +Like before, we also have to deal with range tombstones. However, instead of +dealing with full range tombstones, we generally deal with +`RangeTombstoneMarker` which is just one of the bound of the range tombstone +(so that a range tombstone is composed of 2 "marker" in practice, its start and +its end). I'll discuss the reasoning for this a bit more later. A +`RangeTombstoneMarker` is identified by a `Slice.Bound` (which is to RT markers +what the `Clustering` is to `Row`) and simply store its deletion information. + +The engine thus mainly work with rows and range tombstone markers, and they are +both grouped under the common `Atom` interface. An "unfiltered" is thus just that: +either a row or a range tombstone marker. + +> Side Note: the "Atom" naming is pretty bad. I've reused it mainly because it +> plays a similar role to the existing OnDiskAtom, but it's arguably crappy now +> because a row is definitively not "indivisible". Anyway, renaming suggestions +> are more than welcome. The only alternative I've come up so far are "Block" +> or "Element" but I'm not entirely convinced by either. + +### ClusteringPrefix, Clustering, Slice.Bound and ClusteringComparator + +Atoms are sorted (within a partition). They are ordered by their +`ClusteringPrefix`, which is mainly a common interface for the `Clustering` of +`Row`, and the `Slice.Bound` of `RangeTombstoneMarker`. More generally, a +`ClusteringPrefix` is a prefix of the clustering values for the clustering +columns of the table involved, with a `Clustering` being the special case where +all values are provided. A `Slice.Bound` can be a true prefix however, having +only some of the clustering values. Further, a `Slice.Bound` can be either a +start or end bound, and it can be inclusive or exclusive. Sorting make sure that +a start bound is before anything it "selects" (and conversely for the end). A +`Slice` is then just 2 `Slice.Bound`: a start and a end, and selects anything +that sorts between those. + +`ClusteringPrefix` are compared through the table `ClusteringComparator`, which +is like our existing table comparator except that it only include comparators +for the clustering column values. In particular, it includes neither the +comparator for the column names themselves, nor the post-column-name comparator +for collections (the latter being handled through the `CellPath`, see below). +There is a also a `Clusterable` interface that `Atom` implements and that +simply marks object that can be compared by a `ClusteringComparator`, i.e. +objects that have a `ClusteringPrefix`. + +### Cell + +A cell holds the informations on a single value. It corresponds to a (CQL) +column, has a value, a timestamp and optional ttl and local deletion time. +Further, as said above, complex columns (collections) will have multiple +associated cells. Those cells are distinguished by their `CellPath`, which are +compared through a comparator that depends on the column type. The cells of +simple columns just have a `null` cell path. + +### AtomIterator and PartitionIterator + +As often as possible, atoms are manipulated through iterators. And this through +`AtomIterator`, which is an iterator over the atoms of a single partition, and +`PartitionIterator`, which is an iterator of `AtomIterator`, i.e. an iterator +over multiple partitions. In other words a single partition query fundamentally +returns an `AtomIterator`, while a range query returns a `PartitionIterator`. +Those iterators are closeable and the code has to be make sure to always close +them as they will often have resources to clean (like an OpOrder.Group to close +or files to release). + +The read path mainly consists in getting unfiltered and partition iterators from +sstables and memtable and merging, filtering and transforming them. There is a +number of functions to do just that (merging, filtering and transforming) in +`AtomIterators` and `PartitionIterators`, but there is also a number of classes +(`RowFilteringAtomIterator`, `CountingAtomIterator`, ...) that wraps one of +those iterator type to apply some filtering/transformation. + +`AtomIterator` and `PartitionIterator` also have their doppelgänger +`RowIterator` and `DataIterator` which exists for the sake of making it easier +for the upper layers (StorageProxy and above) to deal with deletion. We'll +discuss those later. + +### Partition: PartitionUpdate, AtomicBTreePartition and CachedPartition + +While we avoid materializing partitions in memory as much as possible (favoring +the iterative approach), there is cases where we need/want to hold some subpart +of a partition in memory and we have a generic `Partition` interface for those. +A `Partition` basically corresponds to materializing an `AtomIterator` and +is thus somewhat equivalent to the existing +`ColumnFamily` (but again, many existing usage of `ColumnFamily` simply use +`AtomIterator`). `Partition` is mainly used through the following +implementations: +* `PartitionUpdate`: this is what a `Mutation` holds and is used to gather + update and apply them to memtables. +* `AtomicBTreePartition`: this is the direct counterpart of AtomicBTreeColumns. + The difference being that the BTree holds rows instead of cells. On updates, we + merge the rows together to create a new one. +* `CachedPartition`: this is used by the row cache. + +### Read commands + +The `ReadCommand` class still exists, but instead of being just for single +partition reads, it's now a common abstract class for both single partition and +range reads. It then has 2 subclass: `SinglePartitionReadCommand` and +`PartitionRangeReadCommand`, the former of which has 2 subclasses itself: +`SinglePartitionSliceCommand` and `SinglePartitionNamesCommand`. All `ReadCommand`, +have a `ColumnFilter` and a `DataLimits` (see below). `PartitionRangeReadCommand` +additionally has a `DataRange`, which is mostly just a range of partition key +with a `PartitionFilter`, while `SinglePartitionReadCommand` has a partition +key and a `PartitionFilter` (see below too). + +The code to execute those queries locally, which used to be in `ColumnFamilyStore` +and `CollationController` is now in those `ReadCommand` classes. For instance, the +`CollationController` code for names queries is in `SinglePartitionNamesCommand`, +and the code to decide if we use a 2ndary index or not is directly in +`ReadCommand.executeLocally()`. + +Note that because they share a common class, all `ReadCommand` actually +return a `PartitionIterator` (an iterator over partitions), even single partition +ones (that iterator will just return one or zero result). It actually allows to +generalize (and simplify) the "response resolver". Instead of having separate resolver +for range and single partition queries, we only have `DataResolver` and +`DigestResolver` that work for any read command. This does mean that the patch +fixes CASSANDRA-2986, and that we could use digest queries for range if we +wanted to (not necessarily saying it's a good idea). + +### ColumnFilter + +`ColumnFilter` is the new `List`. It holds those column restrictions that +can't be directly fulfilled by the `PartitionFilter`, i.e. those that require either a +2ndary index, or filtering. + +### PartitionFilter + +`PartitionFilter` is the new `IDiskAtomFilter`/`QueryFilter`. There is still 2 variants: +`SlicePartitionFilter` and `NamesPartitionFilter`. Both variant includes the actual columns +that are queried (as we don't return full CQL rows anymore), and both can be +reversed. A names filter queries a bunch of rows by names, i.e. has a set of +`Clustering`. A slice filter queries one or more slice of rows. A slice filter +does not however include a limit since that is dealt with by `DataLimits` which +is in `ReadCommand` directly. + + +### DataLimits + +`DataLimits` implement the limits on a query. This is meant to abstract the differences between +how we count for thrift and for CQL. Further, for CQL, this allow to have a limit per partition, +which clean up how DISTINCT queries are handled and allow for CASSANDRA-7017 (the patch doesn't +add support for the PER PARTITION LIMIT syntax of that ticket, but handle it +internally otherwise). + +### SliceableAtomIterator + +The code also use the `SliceableAtomIterator` abstraction. A +`SliceableAtomIterator` is an `AtomIterator` for which we basically know how to seek +into efficiently. In particular, we have a `SSTableIterator` which is a +`SliceableAtomIterator`. That `SSTableIterator` replaces both the existing +`SSTableNamesIterator` and `SSTableSliceIterator`, and the respective +`PartitionFilter` uses that `SliceableAtomIterator` interface to query what +they want exactly. + + +What did that become again? +--------------------------- + +For quick reference, here's the rough correspondence of old classes to new classes: +* `ColumnFamily`: for writes, this is handled by `PartitionUpdate` and + `AtomicBTreePartition`. For reads, this is replaced by `AtomIterator` (or + `RowIterator`, see the parts on tombstones below). For the row cache, there is + a specific `CachedPartition` (which is actually an interface, the implementing + class being `ArrayBackedPartition`) +* `Cell`: there is still a `Cell` class, which is roughly the same thing than + the old one, except that instead of having a cell name, cells are now in a + row and correspond to a column. +* `QueryFilter`: doesn't really exists anymore. What it was holding is now in + `SinglePartitionReadCommand`. +* `AbstractRangeCommand` is now `PartitionRangeReadCommand`. +* `IDiskAtomFilter` is now `PartitionFilter`. +* `List` is now `ColumnFilter`. +* `RowDataResolver`, `RowDigestResolver` and `RangeSliceResponseResolver` are + now `DataResolver` and `DigestResolver`. +* `Row` is now `AtomIterator`. +* `List` is now `PartitionIterator` (or `DataIterator`, see the part about + tombstones below). +* `AbstractCompactedRow` and`LazilyCompactedRow` are not really needed anymore. + Their corresponding code is in `CompactionIterable`. + + +Noteworthy points +----------------- + +### Dealing with tombstones and shadowed cells + +There is a few aspects worth noting regarding the handling of deleted and +shadowed data: +1. it's part of the contract of an `AtomIterator` that it must not shadow it's + own data. In other words, it should not return a cell that is deleted by one + of its own range tombstone, or by its partition level deletion. In practice + this means that we get rid of shadowed data quickly (a good thing) and that + there is a limited amount of places that have to deal with shadowing + (merging being one). +2. Upper layer of the code (anything above StorageProxy) don't care about + deleted data (and deletion informations in general). Basically, as soon as + we've merge results for the replica, we don't need tombstones anymore. + So, instead of requiring all those upper layer to filter tombstones + themselves (which is error prone), we get rid of them as soon as we can, + i.e. as soon as we've resolved replica responses (so in the + `ResponseResolver`). To do that and to make it clear when tombstones have + been filtered (and make the code cleaner), we transform an `AtomIterator` + into a `RowIterator`. Both being essentially the same thing, except that a + `RowIterator` only return live stuffs. Which mean in particular that it's an + iterator of `Row` (since an unfiltered is either a row or a range tombstone and + we've filtered tombstones). Similarly, a `PartitionIterator` becomes a + `DataIterator`, which is just an iterator of `RowIterator`. +3. In the existing code a CQL row deletion involves a range tombstone. But as + row deletions are pretty frequent and range tombstone have inherent + inefficiencies, the patch adds the notion of row deletion, which is just + some optional deletion information on the `Row`. This can be though as just + an optimization of range tombstones that span only a single row. +4. As mentioned at the beginning of this document, the code splits range + tombstones into 2 range tombstone marker, one for each bound. The problem + with storing full range tombstone as we currently do is that it makes it + harder to merge them efficiently, and in particular to "normalize" + overlapping ones. In practice, a given `AtomIterator` guarantees that there + is only one range tombstone to worry about at any given time. In other + words, at any point of the iterator, either there is a single open range + tombstone or there is none, which makes things easier and more efficient. + It's worth noting that we still have the `RangeTombstone` class because for + in-memory structures (`PartitionUpdate`, `AtomicBTreePartition`, ...) we + currently use the existing `RangeTombstoneList` out of convenience. This is + kind of an implementation detail that could change in the future. + +### statics + +Static columns are handled separately from the rest of the rows. In practice, +each `Partition`/`AtomIterator` has a separate "static row" that holds the +value for all the static columns (that row can of course be empty). That row +doesn't really correspond to any CQL row, it's content is simply "merged" to +the result set in `SelectStatement` (very much like we already do). + +### Row liveness + +In CQL, a row can exists even if only its PK columns have values. In other +words, a `Row` can be live even if it doesn't have any cells. Currently, we +handle this through the "row marker" (i.e. through a specific cell). The patch +makes this slightly less hacky by adding a timestamp (and potentially a ttl) to +each row, which can be though as the timestamp (and ttl) of the PK columns. + +Further, when we query only some specific columns in a row, we need to add a +row (with nulls) in the result set if a row is live (it has live cells or the +timestamp we described above) even if it has no actual values for the queried +columns. We currently deal with that by querying entire row all the time, but +the patch change that. This does mean that even when we query only some +columns, we still need to have the information on whether the row itself is +live or not. And because we'll merge results from different sources (which can +include deletions), a boolean is not enough, so we include in a `Row` object +the maximum live timestamp known for this row. Which currently mean that in +practice, we do scan the full row on disk but filter cells we're not interested +by right away (and in fact, we don't even deserialize the value of such cells). +This might be optimizable later on, but expiring data makes that harder (we +typically can't just pre-compute that max live timestamp when writing the +sstable since it can depend on the time of the query). + +### Flyweight pattern + +The patch makes relatively heavy use of a "flyweight"-like pattern. Typically, +`Row` and `Cell` data are stored in arrays, and a given `AtomIterator` will +only use a single `Row` object (and `Cell` object) that points in those arrays +(and thus change at each call to `hasNext()/next()`). This does mean that the +objects returned that way shouldn't be aliased (taken a reference of) without +care. The patch uses an `Aliasable` interface to mark object that may use this +pattern and should thus potentially be copied in the rare cases where a +reference should be kept on them. + +### Reversed queries + +The patch slightly change the way reversed queries are handled. First, while +a reverse query should currently reverse its slices in `SliceQueryFilter`, this +is not the case anymore: `SlicePartitionFilter` always keep its slices in +clustering order and simply handles those from the end to the beginning on +reversed queries. Further, if a query is reversed, the results are returned in +this reverse order all the way through. Which differs from the current code +where `ColumnFamily` actually always holds cells in forward order (making it a +mess for paging and forcing us to re-reverse everything in the result set). + +### Compaction + +As the storage engine works iteratively, compaction simply has to get iterators +on the sstables, merge them and write the result back, along with a simple +filter that skip purgeable tombstones in the process. So there isn't really a +need for `AbstractCompactedRow`/`LazilyCompactedRow` anymore and the +compaction/purging code is now in `CompactionIterable` directly. + +### Short reads + +The current way to handle short reads would require us to consume the whole +result before deciding on a retry (and we currently retry the whole command), +which doesn't work too well in an iterative world. So the patch moves read +protection in `DataResolver` (where it kind of belong anyway) and we don't +retry the full command anymore. Instead, if we realize that a given node has a +short read while its result is consumed, we simply query this node (and this +node only) for a continuation of the result. On top of avoiding the retry of +the whole read (and limiting the number of node queried on the retry), this +also make it trivial to solve CASSANDRA-8933. + + +Storage format (on-disk and on-wire) +------------------------------------ + +Given that the main abstractions are changed, the existing on-wire +`ColumnFamily` format is not appropriate anymore and the patch switches to a +new format. The same can be told of the on-disk format, and while it is not an +objective of CASSANDRA-8099 to get fancy on the on-disk format, using the +on-wire format as on-disk format was actually relatively simple (until more +substantial changes land with CASSANDRA-7447) and the patch does that too. + +For a given partition, the format simply serialize rows one after another +(atoms in practice). For the on-disk format, this means that it is now rows +that are indexed, not cells. The format uses a header that is written at the +beginning of each partition for the on-wire format (in +`AtomIteratorSerializer`) and is kept as a new sstable `Component` for +sstables. The details of the format are described in the javadoc of +`AtomIteratorSerializer` (only used by the on-wire format) and of +`AtomSerializer`, so let me just point the following differences compared to +the current format: +* Clustering values are only serialized once per row (we even skip serializing the + number of elements since that is fixed for a given table). +* Column names are not written for every row. Instead they are written once in + the header. For a given row, we support two small variant: dense and sparse. + When dense, cells come in the order the columns have in the header, meaning + that if a row doesn't have a particular column, this column will still use + a byte. When sparse, we don't have anything if the column doesn't have a cell, + but each cell has an additional 2 bytes which points into the header (so with + vint it should rarely take more than 1 byte in practice). The variant used + is automatically decided based on stats on how many columns set a row has on + average for the source we serialize. +* Values for fixed-width cell values are serialized without a size. +* If a cell has the same timestamp than its row, that timestamp is not repeated + for the cell. Same for the ttl (if applicable). +* Timestamps, ttls and local deletion times are delta encoded so that they are + ripe for vint encoding. The current version of the patch does not yet + activate vint encoding however (neither for on-wire or on-disk). + diff --git a/src/java/org/apache/cassandra/cache/AutoSavingCache.java b/src/java/org/apache/cassandra/cache/AutoSavingCache.java index a204a18fa9..9dda0195d6 100644 --- a/src/java/org/apache/cassandra/cache/AutoSavingCache.java +++ b/src/java/org/apache/cassandra/cache/AutoSavingCache.java @@ -207,7 +207,7 @@ public class AutoSavingCache extends InstrumentingCache if (isSentinel) out.writeLong(((RowCacheSentinel) entry).sentinelId); else - ColumnFamily.serializer.serialize((ColumnFamily) entry, new DataOutputPlusAdapter(out), MessagingService.current_version); + CachedPartition.cacheSerializer.serialize((CachedPartition)entry, new DataOutputPlusAdapter(out)); } public IRowCacheEntry deserialize(DataInput in) throws IOException @@ -167,7 +167,7 @@ public class OHCProvider implements CacheProvider boolean isSentinel = in.readBoolean(); if (isSentinel) return new RowCacheSentinel(in.readLong()); - return ColumnFamily.serializer.deserialize(in, MessagingService.current_version); + return CachedPartition.cacheSerializer.deserialize(in); } public int serializedSize(IRowCacheEntry entry) @@ -177,7 +177,7 @@ public class OHCProvider implements CacheProvider if (entry instanceof RowCacheSentinel) size += typeSizes.sizeof(((RowCacheSentinel) entry).sentinelId); else - size += ColumnFamily.serializer.serializedSize((ColumnFamily) entry, typeSizes, MessagingService.current_version); + size += CachedPartition.cacheSerializer.serializedSize((CachedPartition) entry, typeSizes); return size; } } diff --git a/src/java/org/apache/cassandra/cache/SerializingCacheProvider.java b/src/java/org/apache/cassandra/cache/SerializingCacheProvider.java index f540322917..70d9e7314c 100644 --- a/src/java/org/apache/cassandra/cache/SerializingCacheProvider.java +++ b/src/java/org/apache/cassandra/cache/SerializingCacheProvider.java @@ -21,8 +21,8 @@ import java.io.DataInput; import java.io.IOException; import org.apache.cassandra.config.DatabaseDescriptor; -import org.apache.cassandra.db.ColumnFamily; import org.apache.cassandra.db.TypeSizes; +import org.apache.cassandra.db.partitions.CachedPartition; import org.apache.cassandra.io.ISerializer; import org.apache.cassandra.io.util.DataOutputPlus; import org.apache.cassandra.net.MessagingService; @@ -45,7 +45,7 @@ public class SerializingCacheProvider implements CacheProvider defaultValidator = BytesType.instance; private volatile AbstractType keyValidator = BytesType.instance; private volatile int minCompactionThreshold = DEFAULT_MIN_COMPACTION_THRESHOLD; private volatile int maxCompactionThreshold = DEFAULT_MAX_COMPACTION_THRESHOLD; @@ -186,7 +190,7 @@ public final class CFMetaData private volatile int memtableFlushPeriod = 0; private volatile int defaultTimeToLive = DEFAULT_DEFAULT_TIME_TO_LIVE; private volatile SpeculativeRetry speculativeRetry = DEFAULT_SPECULATIVE_RETRY; - private volatile Map droppedColumns = new HashMap<>(); + private volatile Map droppedColumns = new HashMap(); private volatile Map triggers = new HashMap<>(); private volatile boolean isPurged = false; /* @@ -196,20 +200,18 @@ public final class CFMetaData * clustering key ones, those list are ordered by the "component index" of the * elements. */ - public static final String DEFAULT_KEY_ALIAS = "key"; - public static final String DEFAULT_COLUMN_ALIAS = "column"; - public static final String DEFAULT_VALUE_ALIAS = "value"; - - // We call dense a CF for which each component of the comparator is a clustering column, i.e. no - // component is used to store a regular column names. In other words, non-composite static "thrift" - // and CQL3 CF are *not* dense. - private volatile Boolean isDense; // null means "we don't know and need to infer from other data" private volatile Map columnMetadata = new HashMap<>(); private volatile List partitionKeyColumns; // Always of size keyValidator.componentsCount, null padded if necessary private volatile List clusteringColumns; // Of size comparator.componentsCount or comparator.componentsCount -1, null padded if necessary - private volatile SortedSet regularColumns; // We use a sorted set so iteration is of predictable order (for SELECT for instance) - private volatile SortedSet staticColumns; // Same as above + private volatile PartitionColumns partitionColumns; + + private final boolean isDense; + private final boolean isCompound; + + // For dense tables, this alias the single non-PK column the table contains (since it can only have one). We keep + // that as convenience to access that column more easily (but we could replace calls by partitionColumns().iterator().next() + // for those tables in practice). private volatile ColumnDefinition compactValueColumn; public volatile Class compactionStrategyClass = DEFAULT_COMPACTION_STRATEGY_CLASS; @@ -222,8 +224,6 @@ public final class CFMetaData public CFMetaData readRepairChance(double prop) {readRepairChance = prop; return this;} public CFMetaData dcLocalReadRepairChance(double prop) {dcLocalReadRepairChance = prop; return this;} public CFMetaData gcGraceSeconds(int prop) {gcGraceSeconds = prop; return this;} - public CFMetaData defaultValidator(AbstractType prop) {defaultValidator = prop; return this;} - public CFMetaData keyValidator(AbstractType prop) {keyValidator = prop; return this;} public CFMetaData minCompactionThreshold(int prop) {minCompactionThreshold = prop; return this;} public CFMetaData maxCompactionThreshold(int prop) {maxCompactionThreshold = prop; return this;} public CFMetaData compactionStrategyClass(Class prop) {compactionStrategyClass = prop; return this;} @@ -236,52 +236,120 @@ public final class CFMetaData public CFMetaData memtableFlushPeriod(int prop) {memtableFlushPeriod = prop; return this;} public CFMetaData defaultTimeToLive(int prop) {defaultTimeToLive = prop; return this;} public CFMetaData speculativeRetry(SpeculativeRetry prop) {speculativeRetry = prop; return this;} - public CFMetaData droppedColumns(Map cols) {droppedColumns = cols; return this;} + public CFMetaData droppedColumns(Map cols) {droppedColumns = cols; return this;} public CFMetaData triggers(Map prop) {triggers = prop; return this;} - public CFMetaData isDense(Boolean prop) {isDense = prop; return this;} + + private CFMetaData(String keyspace, + String name, + UUID cfId, + boolean isSuper, + boolean isCounter, + boolean isDense, + boolean isCompound, + List partitionKeyColumns, + List clusteringColumns, + PartitionColumns partitionColumns) + { + this.cfId = cfId; + this.ksName = keyspace; + this.cfName = name; + this.isDense = isDense; + this.isCompound = isCompound; + this.isSuper = isSuper; + this.isCounter = isCounter; + + // A compact table should always have a clustering + assert isCQLTable() || !clusteringColumns.isEmpty() : String.format("For table %s.%s, isDense=%b, isCompound=%b, clustering=%s", ksName, cfName, isDense, isCompound, clusteringColumns); + + this.partitionKeyColumns = partitionKeyColumns; + this.clusteringColumns = clusteringColumns; + this.partitionColumns = partitionColumns; + + this.serializers = new Serializers(this); + rebuild(); + } + + // This rebuild informations that are intrinsically duplicate of the table definition but + // are kept because they are often useful in a different format. + private void rebuild() + { + this.comparator = new ClusteringComparator(extractTypes(clusteringColumns)); + + this.columnMetadata.clear(); + for (ColumnDefinition def : partitionKeyColumns) + this.columnMetadata.put(def.name.bytes, def); + for (ColumnDefinition def : clusteringColumns) + this.columnMetadata.put(def.name.bytes, def); + for (ColumnDefinition def : partitionColumns) + this.columnMetadata.put(def.name.bytes, def); + + List> keyTypes = extractTypes(partitionKeyColumns); + this.keyValidator = keyTypes.size() == 1 ? keyTypes.get(0) : CompositeType.getInstance(keyTypes); + + if (isCompactTable()) + this.compactValueColumn = CompactTables.getCompactValueColumn(partitionColumns, isSuper()); + } + + public static CFMetaData create(String ksName, + String name, + UUID cfId, + boolean isDense, + boolean isCompound, + boolean isSuper, + boolean isCounter, + List columns) + { + List partitions = new ArrayList<>(); + List clusterings = new ArrayList<>(); + PartitionColumns.Builder builder = PartitionColumns.builder(); + + for (ColumnDefinition column : columns) + { + switch (column.kind) + { + case PARTITION_KEY: + partitions.add(column); + break; + case CLUSTERING_COLUMN: + clusterings.add(column); + break; + default: + builder.add(column); + break; + } + } + + Collections.sort(partitions); + Collections.sort(clusterings); + + return new CFMetaData(ksName, + name, + cfId, + isSuper, + isCounter, + isDense, + isCompound, + partitions, + clusterings, + builder.build()); + } + + private static List> extractTypes(List clusteringColumns) + { + List> types = new ArrayList<>(clusteringColumns.size()); + for (ColumnDefinition def : clusteringColumns) + types.add(def.type); + return types; + } /** - * Create new ColumnFamily metadata with generated random ID. - * When loading from existing schema, use CFMetaData - * - * @param keyspace keyspace name - * @param name column family name - * @param comp default comparator + * There is a couple of places in the code where we need a CFMetaData object and don't have one readily available + * and know that only the keyspace and name matter. This creates such "fake" metadata. Use only if you know what + * you're doing. */ - public CFMetaData(String keyspace, String name, ColumnFamilyType type, CellNameType comp) + public static CFMetaData createFake(String keyspace, String name) { - this(keyspace, name, type, comp, UUIDGen.getTimeUUID()); - } - - public CFMetaData(String keyspace, String name, ColumnFamilyType type, CellNameType comp, UUID id) - { - cfId = id; - ksName = keyspace; - cfName = name; - cfType = type; - comparator = comp; - } - - public static CFMetaData denseCFMetaData(String keyspace, String name, AbstractType comp, AbstractType subcc) - { - CellNameType cellNameType = CellNames.fromAbstractType(makeRawAbstractType(comp, subcc), true); - return new CFMetaData(keyspace, name, subcc == null ? ColumnFamilyType.Standard : ColumnFamilyType.Super, cellNameType); - } - - public static CFMetaData sparseCFMetaData(String keyspace, String name, AbstractType comp) - { - CellNameType cellNameType = CellNames.fromAbstractType(comp, false); - return new CFMetaData(keyspace, name, ColumnFamilyType.Standard, cellNameType); - } - - public static CFMetaData denseCFMetaData(String keyspace, String name, AbstractType comp) - { - return denseCFMetaData(keyspace, name, comp, null); - } - - public static AbstractType makeRawAbstractType(AbstractType comparator, AbstractType subComparator) - { - return subComparator == null ? comparator : CompositeType.getInstance(Arrays.asList(comparator, subComparator)); + return CFMetaData.Builder.create(keyspace, name).addPartitionKey("key", BytesType.instance).build(); } public Map getTriggers() @@ -289,14 +357,21 @@ public final class CFMetaData return triggers; } + // Compiles a system metadata public static CFMetaData compile(String cql, String keyspace) { CFStatement parsed = (CFStatement)QueryProcessor.parseStatement(cql); parsed.prepareKeyspace(keyspace); CreateTableStatement statement = (CreateTableStatement) parsed.prepare().statement; - CFMetaData cfm = newSystemMetadata(keyspace, statement.columnFamily(), "", statement.comparator); + CFMetaData.Builder builder = statement.metadataBuilder(); + builder.withId(generateLegacyCfId(keyspace, statement.columnFamily())); + CFMetaData cfm = builder.build(); statement.applyPropertiesTo(cfm); - return cfm.rebuild(); + + return cfm.readRepairChance(0) + .dcLocalReadRepairChance(0) + .gcGraceSeconds(0) + .memtableFlushPeriod(3600 * 1000); } /** @@ -310,26 +385,7 @@ public final class CFMetaData return UUID.nameUUIDFromBytes(ArrayUtils.addAll(ksName.getBytes(), cfName.getBytes())); } - private static CFMetaData newSystemMetadata(String keyspace, String cfName, String comment, CellNameType comparator) - { - return new CFMetaData(keyspace, cfName, ColumnFamilyType.Standard, comparator, generateLegacyCfId(keyspace, cfName)) - .comment(comment) - .readRepairChance(0) - .dcLocalReadRepairChance(0) - .gcGraceSeconds(0) - .memtableFlushPeriod(3600 * 1000); - } - - /** - * Creates CFMetaData for secondary index CF. - * Secondary index CF has the same CF ID as parent's. - * - * @param parent Parent CF where secondary index is created - * @param info Column definition containing secondary index definition - * @param indexComparator Comparator for secondary index - * @return CFMetaData for secondary index - */ - public static CFMetaData newIndexMetadata(CFMetaData parent, ColumnDefinition info, CellNameType indexComparator) + public CFMetaData reloadIndexMetadataProperties(CFMetaData parent) { // Depends on parent's cache setting, turn on its index CF's cache. // Row caching is never enabled; see CASSANDRA-5732 @@ -337,32 +393,21 @@ public final class CFMetaData ? CachingOptions.KEYS_ONLY : CachingOptions.NONE; - return new CFMetaData(parent.ksName, parent.indexColumnFamilyName(info), ColumnFamilyType.Standard, indexComparator, parent.cfId) - .keyValidator(info.type) - .readRepairChance(0.0) - .dcLocalReadRepairChance(0.0) - .gcGraceSeconds(0) - .caching(indexCaching) - .speculativeRetry(parent.speculativeRetry) - .compactionStrategyClass(parent.compactionStrategyClass) - .compactionStrategyOptions(parent.compactionStrategyOptions) - .reloadSecondaryIndexMetadata(parent) - .rebuild(); - } - - public CFMetaData reloadSecondaryIndexMetadata(CFMetaData parent) - { - minCompactionThreshold(parent.minCompactionThreshold); - maxCompactionThreshold(parent.maxCompactionThreshold); - compactionStrategyClass(parent.compactionStrategyClass); - compactionStrategyOptions(parent.compactionStrategyOptions); - compressionParameters(parent.compressionParameters); - return this; + return this.readRepairChance(0.0) + .dcLocalReadRepairChance(0.0) + .gcGraceSeconds(0) + .caching(indexCaching) + .speculativeRetry(parent.speculativeRetry) + .minCompactionThreshold(parent.minCompactionThreshold) + .maxCompactionThreshold(parent.maxCompactionThreshold) + .compactionStrategyClass(parent.compactionStrategyClass) + .compactionStrategyOptions(parent.compactionStrategyOptions) + .compressionParameters(parent.compressionParameters); } public CFMetaData copy() { - return copyOpts(new CFMetaData(ksName, cfName, cfType, comparator, cfId), this); + return copy(cfId); } /** @@ -373,23 +418,42 @@ public final class CFMetaData */ public CFMetaData copy(UUID newCfId) { - return copyOpts(new CFMetaData(ksName, cfName, cfType, comparator, newCfId), this); + return copyOpts(new CFMetaData(ksName, + cfName, + newCfId, + isSuper, + isCounter, + isDense, + isCompound, + copy(partitionKeyColumns), + copy(clusteringColumns), + copy(partitionColumns)), + this); + } + + private static List copy(List l) + { + List copied = new ArrayList<>(l.size()); + for (ColumnDefinition cd : l) + copied.add(cd.copy()); + return copied; + } + + private static PartitionColumns copy(PartitionColumns columns) + { + PartitionColumns.Builder newColumns = PartitionColumns.builder(); + for (ColumnDefinition cd : columns) + newColumns.add(cd.copy()); + return newColumns.build(); } @VisibleForTesting public static CFMetaData copyOpts(CFMetaData newCFMD, CFMetaData oldCFMD) { - List clonedColumns = new ArrayList<>(oldCFMD.allColumns().size()); - for (ColumnDefinition cd : oldCFMD.allColumns()) - clonedColumns.add(cd.copy()); - - return newCFMD.addAllColumnDefinitions(clonedColumns) - .comment(oldCFMD.comment) + return newCFMD.comment(oldCFMD.comment) .readRepairChance(oldCFMD.readRepairChance) .dcLocalReadRepairChance(oldCFMD.dcLocalReadRepairChance) .gcGraceSeconds(oldCFMD.gcGraceSeconds) - .defaultValidator(oldCFMD.defaultValidator) - .keyValidator(oldCFMD.keyValidator) .minCompactionThreshold(oldCFMD.minCompactionThreshold) .maxCompactionThreshold(oldCFMD.maxCompactionThreshold) .compactionStrategyClass(oldCFMD.compactionStrategyClass) @@ -403,9 +467,7 @@ public final class CFMetaData .speculativeRetry(oldCFMD.speculativeRetry) .memtableFlushPeriod(oldCFMD.memtableFlushPeriod) .droppedColumns(new HashMap<>(oldCFMD.droppedColumns)) - .triggers(new HashMap<>(oldCFMD.triggers)) - .isDense(oldCFMD.isDense) - .rebuild(); + .triggers(new HashMap<>(oldCFMD.triggers)); } /** @@ -429,7 +491,7 @@ public final class CFMetaData public boolean isSuper() { - return cfType == ColumnFamilyType.Super; + return isSuper; } /** @@ -476,16 +538,18 @@ public final class CFMetaData return ReadRepairDecision.NONE; } + public AbstractType getColumnDefinitionNameComparator(ColumnDefinition.Kind kind) + { + return (isSuper() && kind == ColumnDefinition.Kind.REGULAR) || (isStaticCompactTable() && kind == ColumnDefinition.Kind.STATIC) + ? thriftColumnNameType() + : UTF8Type.instance; + } + public int getGcGraceSeconds() { return gcGraceSeconds; } - public AbstractType getDefaultValidator() - { - return defaultValidator; - } - public AbstractType getKeyValidator() { return keyValidator; @@ -512,15 +576,21 @@ public final class CFMetaData } // An iterator over all column definitions but that respect the order of a SELECT *. + // This also "hide" the clustering/regular columns for a non-CQL3 non-dense table for backward compatibility + // sake (those are accessible through thrift but not through CQL currently). public Iterator allColumnsInSelectOrder() { + final boolean isStaticCompactTable = isStaticCompactTable(); + final boolean noNonPkColumns = isCompactTable() && CompactTables.hasEmptyCompactValue(this); return new AbstractIterator() { private final Iterator partitionKeyIter = partitionKeyColumns.iterator(); - private final Iterator clusteringIter = clusteringColumns.iterator(); - private boolean valueDone; - private final Iterator staticIter = staticColumns.iterator(); - private final Iterator regularIter = regularColumns.iterator(); + private final Iterator clusteringIter = isStaticCompactTable ? Collections.emptyIterator() : clusteringColumns.iterator(); + private final Iterator otherColumns = noNonPkColumns + ? Collections.emptyIterator() + : (isStaticCompactTable + ? partitionColumns.statics.selectOrderIterator() + : partitionColumns.selectOrderIterator()); protected ColumnDefinition computeNext() { @@ -530,22 +600,7 @@ public final class CFMetaData if (clusteringIter.hasNext()) return clusteringIter.next(); - if (staticIter.hasNext()) - return staticIter.next(); - - if (compactValueColumn != null && !valueDone) - { - valueDone = true; - // If the compactValueColumn is empty, this means we have a dense table but - // with only a PK. As far as selects are concerned, we should ignore the value. - if (compactValueColumn.name.bytes.hasRemaining()) - return compactValueColumn; - } - - if (regularIter.hasNext()) - return regularIter.next(); - - return endOfData(); + return otherColumns.hasNext() ? otherColumns.next() : endOfData(); } }; } @@ -560,19 +615,9 @@ public final class CFMetaData return clusteringColumns; } - public Set regularColumns() + public PartitionColumns partitionColumns() { - return regularColumns; - } - - public Set staticColumns() - { - return staticColumns; - } - - public Iterable regularAndStaticColumns() - { - return Iterables.concat(staticColumns, regularColumns); + return partitionColumns; } public ColumnDefinition compactValueColumn() @@ -580,13 +625,28 @@ public final class CFMetaData return compactValueColumn; } - // TODO: we could use CType for key validation too to make this unnecessary but - // it's unclear it would be a win overall - public CType getKeyValidatorAsCType() + public ClusteringComparator getKeyValidatorAsClusteringComparator() { - return keyValidator instanceof CompositeType - ? new CompoundCType(((CompositeType) keyValidator).types) - : new SimpleCType(keyValidator); + boolean isCompound = keyValidator instanceof CompositeType; + List> types = isCompound + ? ((CompositeType) keyValidator).types + : Collections.>singletonList(keyValidator); + return new ClusteringComparator(types); + } + + public static ByteBuffer serializePartitionKey(ClusteringPrefix keyAsClustering) + { + // TODO: we should stop using Clustering for partition keys. Maybe we can add + // a few methods to DecoratedKey so we don't have to (note that while using a Clustering + // allows to use buildBound(), it's actually used for partition keys only when every restriction + // is an equal, so we could easily create a specific method for keys for that. + if (keyAsClustering.size() == 1) + return keyAsClustering.get(0); + + ByteBuffer[] values = new ByteBuffer[keyAsClustering.size()]; + for (int i = 0; i < keyAsClustering.size(); i++) + values[i] = keyAsClustering.get(i); + return CompositeType.build(values); } public double getBloomFilterFpChance() @@ -627,14 +687,26 @@ public final class CFMetaData return defaultTimeToLive; } - public Map getDroppedColumns() + public Map getDroppedColumns() { return droppedColumns; } - public Boolean getIsDense() + /** + * Returns a "fake" ColumnDefinition corresponding to the dropped column {@code name} + * of {@code null} if there is no such dropped column. + */ + public ColumnDefinition getDroppedColumnDefinition(ByteBuffer name) { - return isDense; + DroppedColumn dropped = droppedColumns.get(name); + if (dropped == null) + return null; + + // We need the type for deserialization purpose. If we don't have the type however, + // it means that it's a dropped column from before 3.0, and in that case using + // BytesType is fine for what we'll be using it for, even if that's a hack. + AbstractType type = dropped.type == null ? BytesType.instance : dropped.type; + return ColumnDefinition.regularDef(this, name, type, null); } @Override @@ -651,13 +723,15 @@ public final class CFMetaData return Objects.equal(cfId, other.cfId) && Objects.equal(ksName, other.ksName) && Objects.equal(cfName, other.cfName) - && Objects.equal(cfType, other.cfType) + && Objects.equal(isDense, other.isDense) + && Objects.equal(isCompound, other.isCompound) + && Objects.equal(isSuper, other.isSuper) + && Objects.equal(isCounter, other.isCounter) && Objects.equal(comparator, other.comparator) && Objects.equal(comment, other.comment) && Objects.equal(readRepairChance, other.readRepairChance) && Objects.equal(dcLocalReadRepairChance, other.dcLocalReadRepairChance) && Objects.equal(gcGraceSeconds, other.gcGraceSeconds) - && Objects.equal(defaultValidator, other.defaultValidator) && Objects.equal(keyValidator, other.keyValidator) && Objects.equal(minCompactionThreshold, other.minCompactionThreshold) && Objects.equal(maxCompactionThreshold, other.maxCompactionThreshold) @@ -673,8 +747,7 @@ public final class CFMetaData && Objects.equal(maxIndexInterval, other.maxIndexInterval) && Objects.equal(speculativeRetry, other.speculativeRetry) && Objects.equal(droppedColumns, other.droppedColumns) - && Objects.equal(triggers, other.triggers) - && Objects.equal(isDense, other.isDense); + && Objects.equal(triggers, other.triggers); } @Override @@ -684,13 +757,15 @@ public final class CFMetaData .append(cfId) .append(ksName) .append(cfName) - .append(cfType) + .append(isDense) + .append(isCompound) + .append(isSuper) + .append(isCounter) .append(comparator) .append(comment) .append(readRepairChance) .append(dcLocalReadRepairChance) .append(gcGraceSeconds) - .append(defaultValidator) .append(keyValidator) .append(minCompactionThreshold) .append(maxCompactionThreshold) @@ -707,16 +782,9 @@ public final class CFMetaData .append(speculativeRetry) .append(droppedColumns) .append(triggers) - .append(isDense) .toHashCode(); } - public AbstractType getValueValidator(CellName cellName) - { - ColumnDefinition def = getColumnDefinition(cellName); - return def == null ? defaultValidator : def.type; - } - /** * Updates this object in place to match the definition in the system schema tables. * @return true if any columns were added, removed, or altered; otherwise, false is returned @@ -739,10 +807,13 @@ public final class CFMetaData validateCompatility(cfm); - // TODO: this method should probably return a new CFMetaData so that - // 1) we can keep comparator final - // 2) updates are applied atomically - comparator = cfm.comparator; + partitionKeyColumns = cfm.partitionKeyColumns; + clusteringColumns = cfm.clusteringColumns; + + boolean hasColumnChange = !partitionColumns.equals(cfm.partitionColumns); + partitionColumns = cfm.partitionColumns; + + rebuild(); // compaction thresholds are checked by ThriftValidation. We shouldn't be doing // validation on the apply path; it's too late for that. @@ -751,7 +822,6 @@ public final class CFMetaData readRepairChance = cfm.readRepairChance; dcLocalReadRepairChance = cfm.dcLocalReadRepairChance; gcGraceSeconds = cfm.gcGraceSeconds; - defaultValidator = cfm.defaultValidator; keyValidator = cfm.keyValidator; minCompactionThreshold = cfm.minCompactionThreshold; maxCompactionThreshold = cfm.maxCompactionThreshold; @@ -767,21 +837,6 @@ public final class CFMetaData if (!cfm.droppedColumns.isEmpty()) droppedColumns = cfm.droppedColumns; - MapDifference columnDiff = Maps.difference(columnMetadata, cfm.columnMetadata); - // columns that are no longer needed - for (ColumnDefinition cd : columnDiff.entriesOnlyOnLeft().values()) - removeColumnDefinition(cd); - // newly added columns - for (ColumnDefinition cd : columnDiff.entriesOnlyOnRight().values()) - addColumnDefinition(cd); - // old columns with updated attributes - for (ByteBuffer name : columnDiff.entriesDiffering().keySet()) - { - ColumnDefinition oldDef = columnMetadata.get(name); - ColumnDefinition def = cfm.columnMetadata.get(name); - addOrReplaceColumnDefinition(oldDef.apply(def)); - } - compactionStrategyClass = cfm.compactionStrategyClass; compactionStrategyOptions = cfm.compactionStrategyOptions; @@ -789,14 +844,9 @@ public final class CFMetaData triggers = cfm.triggers; - isDense(cfm.isDense); - - rebuild(); logger.debug("application result is {}", this); - return !columnDiff.entriesOnlyOnLeft().isEmpty() || - !columnDiff.entriesOnlyOnRight().isEmpty() || - !columnDiff.entriesDiffering().isEmpty(); + return hasColumnChange; } public void validateCompatility(CFMetaData cfm) throws ConfigurationException @@ -812,8 +862,8 @@ public final class CFMetaData throw new ConfigurationException(String.format("Column family ID mismatch (found %s; expected %s)", cfm.cfId, cfId)); - if (cfm.cfType != cfType) - throw new ConfigurationException(String.format("Column family types do not match (found %s; expected %s).", cfm.cfType, cfType)); + if (cfm.isDense != isDense || cfm.isCompound != isCompound || cfm.isCounter != isCounter || cfm.isSuper != isSuper) + throw new ConfigurationException("types do not match."); if (!cfm.comparator.isCompatibleWith(comparator)) throw new ConfigurationException(String.format("Column family comparators do not match or are not compatible (found %s; expected %s).", cfm.comparator.getClass().getSimpleName(), comparator.getClass().getSimpleName())); @@ -891,27 +941,6 @@ public final class CFMetaData return columnMetadata.get(name); } - /** - * Returns a ColumnDefinition given a cell name. - */ - public ColumnDefinition getColumnDefinition(CellName cellName) - { - ColumnIdentifier id = cellName.cql3ColumnName(this); - ColumnDefinition def = id == null - ? getColumnDefinition(cellName.toByteBuffer()) // Means a dense layout, try the full column name - : getColumnDefinition(id); - - // It's possible that the def is a PRIMARY KEY or COMPACT_VALUE one in case a concrete cell - // name conflicts with a CQL column name, which can happen in 2 cases: - // 1) because the user inserted a cell through Thrift that conflicts with a default "alias" used - // by CQL for thrift tables (see #6892). - // 2) for COMPACT STORAGE tables with a single utf8 clustering column, the cell name can be anything, - // including a CQL column name (without this being a problem). - // In any case, this is fine, this just mean that columnDefinition is not the ColumnDefinition we are - // looking for. - return def != null && def.isPartOfCellName() ? def : null; - } - public ColumnDefinition getColumnDefinitionForIndex(String indexName) { for (ColumnDefinition def : allColumns()) @@ -970,21 +999,6 @@ public final class CFMetaData return (cfName + "_" + columnName + "_idx").replaceAll("\\W", ""); } - public Iterator getOnDiskIterator(FileDataInput in, Version version) - { - return getOnDiskIterator(in, ColumnSerializer.Flag.LOCAL, Integer.MIN_VALUE, version); - } - - public Iterator getOnDiskIterator(FileDataInput in, ColumnSerializer.Flag flag, int expireBefore, Version version) - { - return version.getSSTableFormat().getOnDiskIterator(in, flag, expireBefore, this, version); - } - - public AtomDeserializer getOnDiskDeserializer(DataInput in, Version version) - { - return new AtomDeserializer(comparator, in, ColumnSerializer.Flag.LOCAL, Integer.MIN_VALUE, version); - } - public static boolean isNameValid(String name) { return name != null && !name.isEmpty() && name.length() <= Schema.NAME_LENGTH && name.matches("\\w+"); @@ -1004,9 +1018,6 @@ public final class CFMetaData if (!isNameValid(cfName)) throw new ConfigurationException(String.format("ColumnFamily name must not be empty, more than %s characters long, or contain non-alphanumeric-underscore characters (got \"%s\")", Schema.NAME_LENGTH, cfName)); - if (cfType == null) - throw new ConfigurationException(String.format("Invalid column family type for %s", cfName)); - for (int i = 0; i < comparator.size(); i++) { if (comparator.subtype(i) instanceof CounterColumnType) @@ -1016,10 +1027,10 @@ public final class CFMetaData throw new ConfigurationException("CounterColumnType is not a valid key validator"); // Mixing counter with non counter columns is not supported (#2614) - if (defaultValidator instanceof CounterColumnType) + if (isCounter) { - for (ColumnDefinition def : regularAndStaticColumns()) - if (!(def.type instanceof CounterColumnType)) + for (ColumnDefinition def : partitionColumns()) + if (!(def.type instanceof CounterColumnType) && !CompactTables.isSuperColumnMapColumn(def)) throw new ConfigurationException("Cannot add a non counter column (" + def.name + ") in a counter column family"); } else @@ -1040,7 +1051,7 @@ public final class CFMetaData } else { - if (cfType == ColumnFamilyType.Super) + if (isSuper) throw new ConfigurationException("Secondary indexes are not supported on super column families"); if (!isIndexNameValid(c.getIndexName())) throw new ConfigurationException("Illegal index name " + c.getIndexName()); @@ -1117,27 +1128,16 @@ public final class CFMetaData } // The comparator to validate the definition name. - - public AbstractType getColumnDefinitionComparator(ColumnDefinition def) + public AbstractType thriftColumnNameType() { - return getComponentComparator(def.isOnAllComponents() ? null : def.position(), def.kind); - } - - public AbstractType getComponentComparator(Integer componentIndex, ColumnDefinition.Kind kind) - { - switch (kind) + if (isSuper()) { - case REGULAR: - if (componentIndex == null) - return comparator.asAbstractType(); - - AbstractType t = comparator.subtype(componentIndex); - assert t != null : "Non-sensical component index"; - return t; - default: - // CQL3 column names are UTF8 - return UTF8Type.instance; + ColumnDefinition def = compactValueColumn(); + assert def != null && def.type instanceof MapType; + return ((MapType)def.type).nameComparator(); } + + return UTF8Type.instance; } public CFMetaData addAllColumnDefinitions(Collection defs) @@ -1159,17 +1159,43 @@ public final class CFMetaData // know this cannot happen. public CFMetaData addOrReplaceColumnDefinition(ColumnDefinition def) { - if (def.kind == ColumnDefinition.Kind.REGULAR) - comparator.addCQL3Column(def.name); - columnMetadata.put(def.name.bytes, def); + // Adds the definition and rebuild what is necessary. We could call rebuild() but it's not too hard to + // only rebuild the necessary bits. + switch (def.kind) + { + case PARTITION_KEY: + partitionKeyColumns.set(def.position(), def); + List> keyTypes = extractTypes(partitionKeyColumns); + keyValidator = keyTypes.size() == 1 ? keyTypes.get(0) : CompositeType.getInstance(keyTypes); + break; + case CLUSTERING_COLUMN: + clusteringColumns.set(def.position(), def); + comparator = new ClusteringComparator(extractTypes(clusteringColumns)); + break; + case REGULAR: + case STATIC: + PartitionColumns.Builder builder = PartitionColumns.builder(); + for (ColumnDefinition column : partitionColumns) + if (!column.name.equals(def.name)) + builder.add(column); + builder.add(def); + partitionColumns = builder.build(); + // If dense, we must have modified the compact value since that's the only one we can have. + if (isDense) + this.compactValueColumn = def; + break; + } + this.columnMetadata.put(def.name.bytes, def); return this; } public boolean removeColumnDefinition(ColumnDefinition def) { - if (def.kind == ColumnDefinition.Kind.REGULAR) - comparator.removeCQL3Column(def.name); - return columnMetadata.remove(def.name.bytes) != null; + assert !def.isPartitionKey(); + boolean removed = columnMetadata.remove(def.name.bytes) != null; + if (removed) + partitionColumns = partitionColumns.without(def); + return removed; } public void addTriggerDefinition(TriggerDefinition def) throws InvalidRequestException @@ -1192,8 +1218,7 @@ public final class CFMetaData public void recordColumnDrop(ColumnDefinition def) { - assert !def.isOnAllComponents(); - droppedColumns.put(def.name, FBUtilities.timestampMicros()); + droppedColumns.put(def.name, new DroppedColumn(def.type, FBUtilities.timestampMicros())); } public void renameColumn(ColumnIdentifier from, ColumnIdentifier to) throws InvalidRequestException @@ -1205,7 +1230,7 @@ public final class CFMetaData if (getColumnDefinition(to) != null) throw new InvalidRequestException(String.format("Cannot rename column %s to %s in keyspace %s; another column of that name already exist", from, to, cfName)); - if (def.isPartOfCellName()) + if (def.isPartOfCellName(isCQLTable(), isSuper())) { throw new InvalidRequestException(String.format("Cannot rename non PRIMARY KEY part %s", from)); } @@ -1215,188 +1240,37 @@ public final class CFMetaData } ColumnDefinition newDef = def.withNewName(to); - // don't call addColumnDefinition/removeColumnDefition because we want to avoid recomputing - // the CQL3 cfDef between those two operation addOrReplaceColumnDefinition(newDef); - removeColumnDefinition(def); - } - public CFMetaData rebuild() - { - if (isDense == null) - isDense(calculateIsDense(comparator.asAbstractType(), allColumns())); - - List pkCols = nullInitializedList(keyValidator.componentsCount()); - List ckCols = nullInitializedList(comparator.clusteringPrefixSize()); - // We keep things sorted to get consistent/predictable order in select queries - SortedSet regCols = new TreeSet<>(regularColumnComparator); - SortedSet statCols = new TreeSet<>(regularColumnComparator); - ColumnDefinition compactCol = null; - - for (ColumnDefinition def : allColumns()) - { - switch (def.kind) - { - case PARTITION_KEY: - assert !(def.isOnAllComponents() && keyValidator instanceof CompositeType); - pkCols.set(def.position(), def); - break; - case CLUSTERING_COLUMN: - assert !(def.isOnAllComponents() && comparator.isCompound()); - ckCols.set(def.position(), def); - break; - case REGULAR: - regCols.add(def); - break; - case STATIC: - statCols.add(def); - break; - case COMPACT_VALUE: - assert compactCol == null : "There shouldn't be more than one compact value defined: got " + compactCol + " and " + def; - compactCol = def; - break; - } - } - - // Now actually assign the correct value. This is not atomic, but then again, updating CFMetaData is never atomic anyway. - partitionKeyColumns = addDefaultKeyAliases(pkCols); - clusteringColumns = addDefaultColumnAliases(ckCols); - regularColumns = regCols; - staticColumns = statCols; - compactValueColumn = addDefaultValueAlias(compactCol); - return this; - } - - private List addDefaultKeyAliases(List pkCols) - { - for (int i = 0; i < pkCols.size(); i++) - { - if (pkCols.get(i) == null) - { - Integer idx = null; - AbstractType type = keyValidator; - if (keyValidator instanceof CompositeType) - { - idx = i; - type = ((CompositeType)keyValidator).types.get(i); - } - // For compatibility sake, we call the first alias 'key' rather than 'key1'. This - // is inconsistent with column alias, but it's probably not worth risking breaking compatibility now. - ByteBuffer name = ByteBufferUtil.bytes(i == 0 ? DEFAULT_KEY_ALIAS : DEFAULT_KEY_ALIAS + (i + 1)); - ColumnDefinition newDef = ColumnDefinition.partitionKeyDef(this, name, type, idx); - addOrReplaceColumnDefinition(newDef); - pkCols.set(i, newDef); - } - } - return pkCols; - } - - private List addDefaultColumnAliases(List ckCols) - { - for (int i = 0; i < ckCols.size(); i++) - { - if (ckCols.get(i) == null) - { - Integer idx; - AbstractType type; - if (comparator.isCompound()) - { - idx = i; - type = comparator.subtype(i); - } - else - { - idx = null; - type = comparator.asAbstractType(); - } - ByteBuffer name = ByteBufferUtil.bytes(DEFAULT_COLUMN_ALIAS + (i + 1)); - ColumnDefinition newDef = ColumnDefinition.clusteringKeyDef(this, name, type, idx); - addOrReplaceColumnDefinition(newDef); - ckCols.set(i, newDef); - } - } - return ckCols; - } - - private ColumnDefinition addDefaultValueAlias(ColumnDefinition compactValueDef) - { - if (comparator.isDense()) - { - if (compactValueDef != null) - return compactValueDef; - - ColumnDefinition newDef = ColumnDefinition.compactValueDef(this, ByteBufferUtil.bytes(DEFAULT_VALUE_ALIAS), defaultValidator); - addOrReplaceColumnDefinition(newDef); - return newDef; - } + // removeColumnDefinition doesn't work for partition key (expectedly) but renaming one is fine so we still + // want to update columnMetadata. + if (def.isPartitionKey()) + columnMetadata.remove(def.name.bytes); else - { - assert compactValueDef == null; - return null; - } + removeColumnDefinition(def); } - /* - * We call dense a CF for which each component of the comparator is a clustering column, i.e. no - * component is used to store a regular column names. In other words, non-composite static "thrift" - * and CQL3 CF are *not* dense. - * We save whether the table is dense or not during table creation through CQL, but we don't have this - * information for table just created through thrift, nor for table prior to CASSANDRA-7744, so this - * method does its best to infer whether the table is dense or not based on other elements. - */ - public static boolean calculateIsDense(AbstractType comparator, Collection defs) + public boolean isCQLTable() { - /* - * As said above, this method is only here because we need to deal with thrift upgrades. - * Once a CF has been "upgraded", i.e. we've rebuilt and save its CQL3 metadata at least once, - * then we'll have saved the "is_dense" value and will be good to go. - * - * But non-upgraded thrift CF (and pre-7744 CF) will have no value for "is_dense", so we need - * to infer that information without relying on it in that case. And for the most part this is - * easy, a CF that has at least one REGULAR definition is not dense. But the subtlety is that not - * having a REGULAR definition may not mean dense because of CQL3 definitions that have only the - * PRIMARY KEY defined. - * - * So we need to recognize those special case CQL3 table with only a primary key. If we have some - * clustering columns, we're fine as said above. So the only problem is that we cannot decide for - * sure if a CF without REGULAR columns nor CLUSTERING_COLUMN definition is meant to be dense, or if it - * has been created in CQL3 by say: - * CREATE TABLE test (k int PRIMARY KEY) - * in which case it should not be dense. However, we can limit our margin of error by assuming we are - * in the latter case only if the comparator is exactly CompositeType(UTF8Type). - */ - boolean hasRegular = false; - int maxClusteringIdx = -1; - for (ColumnDefinition def : defs) - { - switch (def.kind) - { - case CLUSTERING_COLUMN: - maxClusteringIdx = Math.max(maxClusteringIdx, def.position()); - break; - case REGULAR: - hasRegular = true; - break; - } - } - - return maxClusteringIdx >= 0 - ? maxClusteringIdx == comparator.componentsCount() - 1 - : !hasRegular && !isCQL3OnlyPKComparator(comparator); + return !isSuper() && !isDense() && isCompound(); } - private static boolean isCQL3OnlyPKComparator(AbstractType comparator) + public boolean isCompactTable() { - if (!(comparator instanceof CompositeType)) - return false; - - CompositeType ct = (CompositeType)comparator; - return ct.types.size() == 1 && ct.types.get(0) instanceof UTF8Type; + return !isCQLTable(); } - public boolean isCQL3Table() + public boolean isStaticCompactTable() { - return !isSuper() && !comparator.isDense() && comparator.isCompound(); + return !isSuper && !isDense() && !isCompound(); + } + + private static boolean hasNoNulls(List l) + { + for (T t : l) + if (t == null) + return false; + return true; } private static List nullInitializedList(int size) @@ -1412,34 +1286,50 @@ public final class CFMetaData */ public boolean isThriftCompatible() { - // Super CF are always "thrift compatible". But since they may have defs with a componentIndex != null, - // we have to special case here. - if (isSuper()) - return true; - - for (ColumnDefinition def : allColumns()) - { - // Non-REGULAR ColumnDefinition are not "thrift compatible" per-se, but it's ok because they hold metadata - // this is only of use to CQL3, so we will just skip them in toThrift. - if (def.kind == ColumnDefinition.Kind.REGULAR && !def.isThriftCompatible()) - return false; - } - - // The table might also have no REGULAR columns (be PK-only), but still be "thrift incompatible". See #7832. - if (isCQL3OnlyPKComparator(comparator.asAbstractType()) && !isDense) - return false; - - return true; + return isCompactTable(); } public boolean isCounter() { - return defaultValidator.isCounter(); + return isCounter; } public boolean hasStaticColumns() { - return !staticColumns.isEmpty(); + return !partitionColumns.statics.isEmpty(); + } + + public boolean hasCollectionColumns() + { + for (ColumnDefinition def : partitionColumns()) + if (def.type instanceof CollectionType && def.type.isMultiCell()) + return true; + return false; + } + + // We call dense a CF for which each component of the comparator is a clustering column, i.e. no + // component is used to store a regular column names. In other words, non-composite static "thrift" + // and CQL3 CF are *not* dense. + public boolean isDense() + { + return isDense; + } + + public boolean isCompound() + { + return isCompound; + } + + public Serializers serializers() + { + return serializers; + } + + public AbstractType makeLegacyDefaultValidator() + { + return isCounter() + ? CounterColumnType.instance + : (isCompactTable() ? compactValueColumn().type : BytesType.instance); } @Override @@ -1449,13 +1339,18 @@ public final class CFMetaData .append("cfId", cfId) .append("ksName", ksName) .append("cfName", cfName) - .append("cfType", cfType) + .append("isDense", isDense) + .append("isCompound", isCompound) + .append("isSuper", isSuper) + .append("isCounter", isCounter) .append("comparator", comparator) + .append("partitionColumns", partitionColumns) + .append("partitionKeyColumns", partitionKeyColumns) + .append("clusteringColumns", clusteringColumns) .append("comment", comment) .append("readRepairChance", readRepairChance) .append("dcLocalReadRepairChance", dcLocalReadRepairChance) .append("gcGraceSeconds", gcGraceSeconds) - .append("defaultValidator", defaultValidator) .append("keyValidator", keyValidator) .append("minCompactionThreshold", minCompactionThreshold) .append("maxCompactionThreshold", maxCompactionThreshold) @@ -1472,7 +1367,234 @@ public final class CFMetaData .append("speculativeRetry", speculativeRetry) .append("droppedColumns", droppedColumns) .append("triggers", triggers.values()) - .append("isDense", isDense) .toString(); } + + public static class Builder + { + private final String keyspace; + private final String table; + private final boolean isDense; + private final boolean isCompound; + private final boolean isSuper; + private final boolean isCounter; + + private UUID tableId; + + private final List> partitionKeys = new ArrayList<>(); + private final List> clusteringColumns = new ArrayList<>(); + private final List> staticColumns = new ArrayList<>(); + private final List> regularColumns = new ArrayList<>(); + + private Builder(String keyspace, String table, boolean isDense, boolean isCompound, boolean isSuper, boolean isCounter) + { + this.keyspace = keyspace; + this.table = table; + this.isDense = isDense; + this.isCompound = isCompound; + this.isSuper = isSuper; + this.isCounter = isCounter; + } + + public static Builder create(String keyspace, String table) + { + return create(keyspace, table, false, true, false); + } + + public static Builder create(String keyspace, String table, boolean isDense, boolean isCompound, boolean isCounter) + { + return create(keyspace, table, isDense, isCompound, false, isCounter); + } + + public static Builder create(String keyspace, String table, boolean isDense, boolean isCompound, boolean isSuper, boolean isCounter) + { + return new Builder(keyspace, table, isDense, isCompound, isSuper, isCounter); + } + + public static Builder createDense(String keyspace, String table, boolean isCompound, boolean isCounter) + { + return create(keyspace, table, true, isCompound, isCounter); + } + + public static Builder createSuper(String keyspace, String table, boolean isCounter) + { + return create(keyspace, table, false, false, true, isCounter); + } + + public Builder withId(UUID tableId) + { + this.tableId = tableId; + return this; + } + + public Builder addPartitionKey(String name, AbstractType type) + { + return addPartitionKey(ColumnIdentifier.getInterned(name, false), type); + } + + public Builder addPartitionKey(ColumnIdentifier name, AbstractType type) + { + this.partitionKeys.add(Pair.create(name, type)); + return this; + } + + public Builder addClusteringColumn(String name, AbstractType type) + { + return addClusteringColumn(ColumnIdentifier.getInterned(name, false), type); + } + + public Builder addClusteringColumn(ColumnIdentifier name, AbstractType type) + { + this.clusteringColumns.add(Pair.create(name, type)); + return this; + } + + public Builder addRegularColumn(String name, AbstractType type) + { + return addRegularColumn(ColumnIdentifier.getInterned(name, false), type); + } + + public Builder addRegularColumn(ColumnIdentifier name, AbstractType type) + { + this.regularColumns.add(Pair.create(name, type)); + return this; + } + + public boolean hasRegulars() + { + return !this.regularColumns.isEmpty(); + } + + public Builder addStaticColumn(String name, AbstractType type) + { + return addStaticColumn(ColumnIdentifier.getInterned(name, false), type); + } + + public Builder addStaticColumn(ColumnIdentifier name, AbstractType type) + { + this.staticColumns.add(Pair.create(name, type)); + return this; + } + + public Set usedColumnNames() + { + Set usedNames = new HashSet<>(); + for (Pair p : partitionKeys) + usedNames.add(p.left.toString()); + for (Pair p : clusteringColumns) + usedNames.add(p.left.toString()); + for (Pair p : staticColumns) + usedNames.add(p.left.toString()); + for (Pair p : regularColumns) + usedNames.add(p.left.toString()); + return usedNames; + } + + public CFMetaData build() + { + if (tableId == null) + tableId = UUIDGen.getTimeUUID(); + + List partitions = new ArrayList<>(partitionKeys.size()); + List clusterings = new ArrayList<>(clusteringColumns.size()); + PartitionColumns.Builder builder = PartitionColumns.builder(); + + for (int i = 0; i < partitionKeys.size(); i++) + { + Pair p = partitionKeys.get(i); + Integer componentIndex = partitionKeys.size() == 1 ? null : i; + partitions.add(new ColumnDefinition(keyspace, table, p.left, p.right, componentIndex, ColumnDefinition.Kind.PARTITION_KEY)); + } + + for (int i = 0; i < clusteringColumns.size(); i++) + { + Pair p = clusteringColumns.get(i); + clusterings.add(new ColumnDefinition(keyspace, table, p.left, p.right, i, ColumnDefinition.Kind.CLUSTERING_COLUMN)); + } + + for (int i = 0; i < regularColumns.size(); i++) + { + Pair p = regularColumns.get(i); + builder.add(new ColumnDefinition(keyspace, table, p.left, p.right, null, ColumnDefinition.Kind.REGULAR)); + } + + for (int i = 0; i < staticColumns.size(); i++) + { + Pair p = staticColumns.get(i); + builder.add(new ColumnDefinition(keyspace, table, p.left, p.right, null, ColumnDefinition.Kind.STATIC)); + } + + return new CFMetaData(keyspace, + table, + tableId, + isSuper, + isCounter, + isDense, + isCompound, + partitions, + clusterings, + builder.build()); + } + } + + // We don't use UUIDSerializer below because we want to use this with vint-encoded streams and UUIDSerializer + // currently assumes a NATIVE encoding. This is also why we don't use writeLong()/readLong in the methods below: + // this would encode the values, but by design of UUID it is likely that both long will be very big numbers + // and so we have a fair change that the encoding will take more than 16 bytes which is not desirable. Note that + // we could make UUIDSerializer work as the serializer below, but I'll keep that to later. + public static class Serializer + { + private static void writeLongAsSeparateBytes(long value, DataOutputPlus out) throws IOException + { + for (int i = 7; i >= 0; i--) + out.writeByte((int)((value >> (8 * i)) & 0xFF)); + } + + private static long readLongAsSeparateBytes(DataInput in) throws IOException + { + long val = 0; + for (int i = 7; i >= 0; i--) + val |= ((long)in.readUnsignedByte()) << (8 * i); + return val; + } + + public void serialize(CFMetaData metadata, DataOutputPlus out, int version) throws IOException + { + writeLongAsSeparateBytes(metadata.cfId.getMostSignificantBits(), out); + writeLongAsSeparateBytes(metadata.cfId.getLeastSignificantBits(), out); + } + + public CFMetaData deserialize(DataInput in, int version) throws IOException + { + UUID cfId = new UUID(readLongAsSeparateBytes(in), readLongAsSeparateBytes(in)); + CFMetaData metadata = Schema.instance.getCFMetaData(cfId); + if (metadata == null) + { + String message = String.format("Couldn't find table for cfId %s. If a table was just " + + "created, this is likely due to the schema not being fully propagated. Please wait for schema " + + "agreement on table creation.", cfId); + throw new UnknownColumnFamilyException(message, cfId); + } + + return metadata; + } + + public long serializedSize(CFMetaData metadata, int version, TypeSizes sizes) + { + // We've made sure it was encoded as 16 bytes whatever the TypeSizes is. + return 16; + } + } + + public static class DroppedColumn + { + public final AbstractType type; + public final long droppedTime; + + public DroppedColumn(AbstractType type, long droppedTime) + { + this.type = type; + this.droppedTime = droppedTime; + } + } } diff --git a/src/java/org/apache/cassandra/config/ColumnDefinition.java b/src/java/org/apache/cassandra/config/ColumnDefinition.java index b33718fc81..ea008168d2 100644 --- a/src/java/org/apache/cassandra/config/ColumnDefinition.java +++ b/src/java/org/apache/cassandra/config/ColumnDefinition.java @@ -26,18 +26,19 @@ import com.google.common.base.Objects; import com.google.common.collect.Lists; import org.apache.cassandra.cql3.*; +import org.apache.cassandra.db.rows.*; import org.apache.cassandra.db.marshal.*; +import org.apache.cassandra.serializers.MarshalException; import org.apache.cassandra.exceptions.*; +import org.apache.cassandra.utils.ByteBufferUtil; -public class ColumnDefinition extends ColumnSpecification +public class ColumnDefinition extends ColumnSpecification implements Comparable { /* * The type of CQL3 column this definition represents. - * There is 3 main type of CQL3 columns: those parts of the partition key, - * those parts of the clustering key and the other, regular ones. - * But when COMPACT STORAGE is used, there is by design only one regular - * column, whose name is not stored in the data contrarily to the column of - * type REGULAR. Hence the COMPACT_VALUE type to distinguish it below. + * There is 4 main type of CQL3 columns: those parts of the partition key, + * those parts of the clustering key and amongst the others, regular and + * static ones. * * Note that thrift only knows about definitions of type REGULAR (and * the ones whose componentIndex == null). @@ -47,8 +48,12 @@ public class ColumnDefinition extends ColumnSpecification PARTITION_KEY, CLUSTERING_COLUMN, REGULAR, - STATIC, - COMPACT_VALUE + STATIC; + + public boolean isPrimaryKeyKind() + { + return this == PARTITION_KEY || this == CLUSTERING_COLUMN; + } } public final Kind kind; @@ -64,14 +69,17 @@ public class ColumnDefinition extends ColumnSpecification */ private final Integer componentIndex; + private final Comparator cellPathComparator; + private final Comparator cellComparator; + public static ColumnDefinition partitionKeyDef(CFMetaData cfm, ByteBuffer name, AbstractType validator, Integer componentIndex) { return new ColumnDefinition(cfm, name, validator, componentIndex, Kind.PARTITION_KEY); } - public static ColumnDefinition partitionKeyDef(String ksName, String cfName, ByteBuffer name, AbstractType validator, Integer componentIndex) + public static ColumnDefinition partitionKeyDef(String ksName, String cfName, String name, AbstractType validator, Integer componentIndex) { - return new ColumnDefinition(ksName, cfName, new ColumnIdentifier(name, UTF8Type.instance), validator, null, null, null, componentIndex, Kind.PARTITION_KEY); + return new ColumnDefinition(ksName, cfName, ColumnIdentifier.getInterned(name, true), validator, null, null, null, componentIndex, Kind.PARTITION_KEY); } public static ColumnDefinition clusteringKeyDef(CFMetaData cfm, ByteBuffer name, AbstractType validator, Integer componentIndex) @@ -79,26 +87,31 @@ public class ColumnDefinition extends ColumnSpecification return new ColumnDefinition(cfm, name, validator, componentIndex, Kind.CLUSTERING_COLUMN); } + public static ColumnDefinition clusteringKeyDef(String ksName, String cfName, String name, AbstractType validator, Integer componentIndex) + { + return new ColumnDefinition(ksName, cfName, ColumnIdentifier.getInterned(name, true), validator, null, null, null, componentIndex, Kind.CLUSTERING_COLUMN); + } + public static ColumnDefinition regularDef(CFMetaData cfm, ByteBuffer name, AbstractType validator, Integer componentIndex) { return new ColumnDefinition(cfm, name, validator, componentIndex, Kind.REGULAR); } + public static ColumnDefinition regularDef(String ksName, String cfName, String name, AbstractType validator, Integer componentIndex) + { + return new ColumnDefinition(ksName, cfName, ColumnIdentifier.getInterned(name, true), validator, null, null, null, componentIndex, Kind.REGULAR); + } + public static ColumnDefinition staticDef(CFMetaData cfm, ByteBuffer name, AbstractType validator, Integer componentIndex) { return new ColumnDefinition(cfm, name, validator, componentIndex, Kind.STATIC); } - public static ColumnDefinition compactValueDef(CFMetaData cfm, ByteBuffer name, AbstractType validator) - { - return new ColumnDefinition(cfm, name, validator, null, Kind.COMPACT_VALUE); - } - public ColumnDefinition(CFMetaData cfm, ByteBuffer name, AbstractType validator, Integer componentIndex, Kind kind) { this(cfm.ksName, cfm.cfName, - new ColumnIdentifier(name, cfm.getComponentComparator(componentIndex, kind)), + ColumnIdentifier.getInterned(name, cfm.getColumnDefinitionNameComparator(kind)), validator, null, null, @@ -107,6 +120,11 @@ public class ColumnDefinition extends ColumnSpecification kind); } + public ColumnDefinition(String ksName, String cfName, ColumnIdentifier name, AbstractType type, Integer componentIndex, Kind kind) + { + this(ksName, cfName, name, type, null, null, null, componentIndex, kind); + } + @VisibleForTesting public ColumnDefinition(String ksName, String cfName, @@ -119,11 +137,55 @@ public class ColumnDefinition extends ColumnSpecification Kind kind) { super(ksName, cfName, name, validator); - assert name != null && validator != null; + assert name != null && validator != null && kind != null; + assert name.isInterned(); this.kind = kind; this.indexName = indexName; this.componentIndex = componentIndex; this.setIndexType(indexType, indexOptions); + this.cellPathComparator = makeCellPathComparator(kind, validator); + this.cellComparator = makeCellComparator(cellPathComparator); + } + + private static Comparator makeCellPathComparator(Kind kind, AbstractType validator) + { + if (kind.isPrimaryKeyKind() || !validator.isCollection() || !validator.isMultiCell()) + return null; + + final CollectionType type = (CollectionType)validator; + return new Comparator() + { + public int compare(CellPath path1, CellPath path2) + { + if (path1.size() == 0 || path2.size() == 0) + { + if (path1 == CellPath.BOTTOM) + return path2 == CellPath.BOTTOM ? 0 : -1; + if (path1 == CellPath.TOP) + return path2 == CellPath.TOP ? 0 : 1; + return path2 == CellPath.BOTTOM ? 1 : -1; + } + + // This will get more complicated once we have non-frozen UDT and nested collections + assert path1.size() == 1 && path2.size() == 1; + return type.nameComparator().compare(path1.get(0), path2.get(0)); + } + }; + } + + private static Comparator makeCellComparator(final Comparator cellPathComparator) + { + return new Comparator() + { + public int compare(Cell c1, Cell c2) + { + int cmp = c1.column().compareTo(c2.column()); + if (cmp != 0 || cellPathComparator == null) + return cmp; + + return cellPathComparator.compare(c1.path(), c2.path()); + } + }; } public ColumnDefinition copy() @@ -166,11 +228,6 @@ public class ColumnDefinition extends ColumnSpecification return kind == Kind.REGULAR; } - public boolean isCompactValue() - { - return kind == Kind.COMPACT_VALUE; - } - // The componentIndex. This never return null however for convenience sake: // if componentIndex == null, this return 0. So caller should first check // isOnAllComponents() to distinguish if that's a possibility. @@ -220,23 +277,26 @@ public class ColumnDefinition extends ColumnSpecification .toString(); } - public boolean isThriftCompatible() - { - return kind == ColumnDefinition.Kind.REGULAR && componentIndex == null; - } - public boolean isPrimaryKeyColumn() { - return kind == Kind.PARTITION_KEY || kind == Kind.CLUSTERING_COLUMN; + return kind.isPrimaryKeyKind(); } /** * Whether the name of this definition is serialized in the cell nane, i.e. whether * it's not just a non-stored CQL metadata. */ - public boolean isPartOfCellName() + public boolean isPartOfCellName(boolean isCQL3Table, boolean isSuper) { - return kind == Kind.REGULAR || kind == Kind.STATIC; + // When converting CQL3 tables to thrift, any regular or static column ends up in the cell name. + // When it's a compact table however, the REGULAR definition is the name for the cell value of "dynamic" + // column (so it's not part of the cell name) and it's static columns that ends up in the cell name. + if (isCQL3Table) + return kind == Kind.REGULAR || kind == Kind.STATIC; + else if (isSuper) + return kind == Kind.REGULAR; + else + return kind == Kind.STATIC; } public ColumnDefinition apply(ColumnDefinition def) throws ConfigurationException @@ -333,4 +393,87 @@ public class ColumnDefinition extends ColumnSpecification } }); } + + public int compareTo(ColumnDefinition other) + { + if (this == other) + return 0; + + if (kind != other.kind) + return kind.ordinal() < other.kind.ordinal() ? -1 : 1; + if (position() != other.position()) + return position() < other.position() ? -1 : 1; + + if (isStatic() != other.isStatic()) + return isStatic() ? -1 : 1; + if (isComplex() != other.isComplex()) + return isComplex() ? 1 : -1; + + return ByteBufferUtil.compareUnsigned(name.bytes, other.name.bytes); + } + + public Comparator cellPathComparator() + { + return cellPathComparator; + } + + public Comparator cellComparator() + { + return cellComparator; + } + + public boolean isComplex() + { + return cellPathComparator != null; + } + + public CellPath.Serializer cellPathSerializer() + { + // Collections are our only complex so far, so keep it simple + return CollectionType.cellPathSerializer; + } + + public void validateCellValue(ByteBuffer value) + { + type.validateCellValue(value); + } + + public void validateCellPath(CellPath path) + { + if (!isComplex()) + throw new MarshalException("Only complex cells should have a cell path"); + + assert type instanceof CollectionType; + ((CollectionType)type).nameComparator().validate(path.get(0)); + } + + public static String toCQLString(Iterable defs) + { + return toCQLString(defs.iterator()); + } + + public static String toCQLString(Iterator defs) + { + if (!defs.hasNext()) + return ""; + + StringBuilder sb = new StringBuilder(); + sb.append(defs.next().name); + while (defs.hasNext()) + sb.append(", ").append(defs.next().name); + return sb.toString(); + } + + /** + * The type of the cell values for cell belonging to this column. + * + * This is the same than the column type, except for collections where it's the 'valueComparator' + * of the collection. + */ + public AbstractType cellValueType() + { + return type instanceof CollectionType + ? ((CollectionType)type).valueComparator() + : type; + } } diff --git a/src/java/org/apache/cassandra/config/DatabaseDescriptor.java b/src/java/org/apache/cassandra/config/DatabaseDescriptor.java index 913d23c4d0..f0dda099db 100644 --- a/src/java/org/apache/cassandra/config/DatabaseDescriptor.java +++ b/src/java/org/apache/cassandra/config/DatabaseDescriptor.java @@ -1341,9 +1341,15 @@ public class DatabaseDescriptor } @VisibleForTesting - public static void setAutoSnapshot(boolean autoSnapshot) { + public static void setAutoSnapshot(boolean autoSnapshot) + { conf.auto_snapshot = autoSnapshot; } + @VisibleForTesting + public static boolean getAutoSnapshot() + { + return conf.auto_snapshot; + } public static boolean isAutoBootstrap() { diff --git a/src/java/org/apache/cassandra/config/YamlConfigurationLoader.java b/src/java/org/apache/cassandra/config/YamlConfigurationLoader.java index 8331f85e8d..062f64dee7 100644 --- a/src/java/org/apache/cassandra/config/YamlConfigurationLoader.java +++ b/src/java/org/apache/cassandra/config/YamlConfigurationLoader.java @@ -70,8 +70,11 @@ public class YamlConfigurationLoader implements ConfigurationLoader { String required = "file:" + File.separator + File.separator; if (!configUrl.startsWith(required)) - throw new ConfigurationException("Expecting URI in variable: [cassandra.config]. Please prefix the file with " + required + File.separator + - " for local files or " + required + "" + File.separator + " for remote files. Aborting. If you are executing this from an external tool, it needs to set Config.setClientMode(true) to avoid loading configuration."); + throw new ConfigurationException(String.format( + "Expecting URI in variable: [cassandra.config]. Found[%s]. Please prefix the file with [%s%s] for local " + + "files and [%s%s] for remote files. If you are executing this from an external tool, it needs " + + "to set Config.setClientMode(true) to avoid loading configuration.", + configUrl, required, File.separator, required, File.separator)); throw new ConfigurationException("Cannot locate " + configUrl + ". If this is a local file, please confirm you've provided " + required + File.separator + " as a URI prefix."); } } diff --git a/src/java/org/apache/cassandra/cql3/Attributes.java b/src/java/org/apache/cassandra/cql3/Attributes.java index 7b38e9faee..97bdcd1d97 100644 --- a/src/java/org/apache/cassandra/cql3/Attributes.java +++ b/src/java/org/apache/cassandra/cql3/Attributes.java @@ -23,7 +23,6 @@ import java.util.Collections; import com.google.common.collect.Iterables; import org.apache.cassandra.cql3.functions.Function; -import org.apache.cassandra.db.ExpiringCell; import org.apache.cassandra.db.marshal.Int32Type; import org.apache.cassandra.db.marshal.LongType; import org.apache.cassandra.exceptions.InvalidRequestException; @@ -36,6 +35,8 @@ import org.apache.cassandra.utils.ByteBufferUtil; */ public class Attributes { + public static final int MAX_TTL = 20 * 365 * 24 * 60 * 60; // 20 years in seconds + private final Term timestamp; private final Term timeToLive; @@ -121,8 +122,8 @@ public class Attributes if (ttl < 0) throw new InvalidRequestException("A TTL must be greater or equal to 0, but was " + ttl); - if (ttl > ExpiringCell.MAX_TTL) - throw new InvalidRequestException(String.format("ttl is too large. requested (%d) maximum (%d)", ttl, ExpiringCell.MAX_TTL)); + if (ttl > MAX_TTL) + throw new InvalidRequestException(String.format("ttl is too large. requested (%d) maximum (%d)", ttl, MAX_TTL)); return ttl; } diff --git a/src/java/org/apache/cassandra/cql3/ColumnCondition.java b/src/java/org/apache/cassandra/cql3/ColumnCondition.java index c7b5ddb12c..50f79f4c79 100644 --- a/src/java/org/apache/cassandra/cql3/ColumnCondition.java +++ b/src/java/org/apache/cassandra/cql3/ColumnCondition.java @@ -20,18 +20,12 @@ package org.apache.cassandra.cql3; import java.nio.ByteBuffer; import java.util.*; -import com.google.common.base.Predicate; import com.google.common.collect.Iterables; import com.google.common.collect.Iterators; import org.apache.cassandra.config.ColumnDefinition; import org.apache.cassandra.cql3.functions.Function; -import org.apache.cassandra.db.Cell; -import org.apache.cassandra.db.ColumnFamily; -import org.apache.cassandra.db.composites.CellName; -import org.apache.cassandra.db.composites.CellNameType; -import org.apache.cassandra.db.composites.Composite; -import org.apache.cassandra.db.filter.ColumnSlice; +import org.apache.cassandra.db.rows.*; import org.apache.cassandra.db.marshal.*; import org.apache.cassandra.exceptions.InvalidRequestException; import org.apache.cassandra.transport.Server; @@ -160,20 +154,19 @@ public class ColumnCondition /** * Validates whether this condition applies to {@code current}. */ - public abstract boolean appliesTo(Composite rowPrefix, ColumnFamily current, long now) throws InvalidRequestException; + public abstract boolean appliesTo(Row row) throws InvalidRequestException; public ByteBuffer getCollectionElementValue() { return null; } - protected boolean isSatisfiedByValue(ByteBuffer value, Cell c, AbstractType type, Operator operator, long now) throws InvalidRequestException + protected boolean isSatisfiedByValue(ByteBuffer value, Cell c, AbstractType type, Operator operator) throws InvalidRequestException { - ByteBuffer columnValue = (c == null || !c.isLive(now)) ? null : c.value(); - return compareWithOperator(operator, type, value, columnValue); + return compareWithOperator(operator, type, value, c == null ? null : c.value()); } - /** Returns true if the operator is satisfied (i.e. "value operator otherValue == true"), false otherwise. */ + /** Returns true if the operator is satisfied (i.e. "otherValue operator value == true"), false otherwise. */ protected boolean compareWithOperator(Operator operator, AbstractType type, ByteBuffer value, ByteBuffer otherValue) throws InvalidRequestException { if (value == ByteBufferUtil.UNSET_BYTE_BUFFER) @@ -195,41 +188,29 @@ public class ColumnCondition // the condition value is not null, so only NEQ can return true return operator == Operator.NEQ; } - int comparison = type.compare(otherValue, value); - switch (operator) - { - case EQ: - return comparison == 0; - case LT: - return comparison < 0; - case LTE: - return comparison <= 0; - case GT: - return comparison > 0; - case GTE: - return comparison >= 0; - case NEQ: - return comparison != 0; - default: - // we shouldn't get IN, CONTAINS, or CONTAINS KEY here - throw new AssertionError(); - } + return operator.isSatisfiedBy(type, otherValue, value); } + } - protected Iterator collectionColumns(CellName collection, ColumnFamily cf, final long now) - { - // We are testing for collection equality, so we need to have the expected values *and* only those. - ColumnSlice[] collectionSlice = new ColumnSlice[]{ collection.slice() }; - // Filter live columns, this makes things simpler afterwards - return Iterators.filter(cf.iterator(collectionSlice), new Predicate() - { - public boolean apply(Cell c) - { - // we only care about live columns - return c.isLive(now); - } - }); - } + private static Cell getCell(Row row, ColumnDefinition column) + { + // If we're asking for a given cell, and we didn't got any row from our read, it's + // the same as not having said cell. + return row == null ? null : row.getCell(column); + } + + private static Cell getCell(Row row, ColumnDefinition column, CellPath path) + { + // If we're asking for a given cell, and we didn't got any row from our read, it's + // the same as not having said cell. + return row == null ? null : row.getCell(column, path); + } + + private static Iterator getCells(Row row, ColumnDefinition column) + { + // If we're asking for a complex cells, and we didn't got any row from our read, it's + // the same as not having any cells for that column. + return row == null ? Collections.emptyIterator() : row.getCells(column); } /** @@ -247,10 +228,9 @@ public class ColumnCondition this.value = condition.value.bindAndGet(options); } - public boolean appliesTo(Composite rowPrefix, ColumnFamily current, long now) throws InvalidRequestException + public boolean appliesTo(Row row) throws InvalidRequestException { - CellName name = current.metadata().comparator.create(rowPrefix, column); - return isSatisfiedByValue(value, current.getColumn(name), column.type, operator, now); + return isSatisfiedByValue(value, getCell(row, column), column.type, operator); } } @@ -276,12 +256,12 @@ public class ColumnCondition } } - public boolean appliesTo(Composite rowPrefix, ColumnFamily current, long now) throws InvalidRequestException + public boolean appliesTo(Row row) throws InvalidRequestException { - CellName name = current.metadata().comparator.create(rowPrefix, column); + Cell c = getCell(row, column); for (ByteBuffer value : inValues) { - if (isSatisfiedByValue(value, current.getColumn(name), column.type, Operator.EQ, now)) + if (isSatisfiedByValue(value, c, column.type, Operator.EQ)) return true; } return false; @@ -303,7 +283,7 @@ public class ColumnCondition this.value = condition.value.bindAndGet(options); } - public boolean appliesTo(Composite rowPrefix, ColumnFamily current, final long now) throws InvalidRequestException + public boolean appliesTo(Row row) throws InvalidRequestException { if (collectionElement == null) throw new InvalidRequestException("Invalid null value for " + (column.type instanceof MapType ? "map" : "list") + " element access"); @@ -313,14 +293,13 @@ public class ColumnCondition MapType mapType = (MapType) column.type; if (column.type.isMultiCell()) { - Cell cell = current.getColumn(current.metadata().comparator.create(rowPrefix, column, collectionElement)); - return isSatisfiedByValue(value, cell, mapType.getValuesType(), operator, now); + Cell cell = getCell(row, column, CellPath.create(collectionElement)); + return isSatisfiedByValue(value, cell, ((MapType) column.type).getValuesType(), operator); } else { - Cell cell = current.getColumn(current.metadata().comparator.create(rowPrefix, column)); - ByteBuffer mapElementValue = cell.isLive(now) ? mapType.getSerializer().getSerializedValue(cell.value(), collectionElement, mapType.getKeysType()) - : null; + Cell cell = getCell(row, column); + ByteBuffer mapElementValue = mapType.getSerializer().getSerializedValue(cell.value(), collectionElement, mapType.getKeysType()); return compareWithOperator(operator, mapType.getValuesType(), value, mapElementValue); } } @@ -329,16 +308,13 @@ public class ColumnCondition ListType listType = (ListType) column.type; if (column.type.isMultiCell()) { - ByteBuffer columnValue = getListItem( - collectionColumns(current.metadata().comparator.create(rowPrefix, column), current, now), - getListIndex(collectionElement)); - return compareWithOperator(operator, listType.getElementsType(), value, columnValue); + ByteBuffer columnValue = getListItem(getCells(row, column), getListIndex(collectionElement)); + return compareWithOperator(operator, ((ListType)column.type).getElementsType(), value, columnValue); } else { - Cell cell = current.getColumn(current.metadata().comparator.create(rowPrefix, column)); - ByteBuffer listElementValue = cell.isLive(now) ? listType.getSerializer().getElement(cell.value(), getListIndex(collectionElement)) - : null; + Cell cell = getCell(row, column); + ByteBuffer listElementValue = listType.getSerializer().getElement(cell.value(), getListIndex(collectionElement)); return compareWithOperator(operator, listType.getElementsType(), value, listElementValue); } } @@ -387,33 +363,31 @@ public class ColumnCondition } } - public boolean appliesTo(Composite rowPrefix, ColumnFamily current, final long now) throws InvalidRequestException + public boolean appliesTo(Row row) throws InvalidRequestException { if (collectionElement == null) throw new InvalidRequestException("Invalid null value for " + (column.type instanceof MapType ? "map" : "list") + " element access"); - CellNameType nameType = current.metadata().comparator; if (column.type instanceof MapType) { MapType mapType = (MapType) column.type; AbstractType valueType = mapType.getValuesType(); if (column.type.isMultiCell()) { - CellName name = nameType.create(rowPrefix, column, collectionElement); - Cell item = current.getColumn(name); + Cell item = getCell(row, column, CellPath.create(collectionElement)); for (ByteBuffer value : inValues) { - if (isSatisfiedByValue(value, item, valueType, Operator.EQ, now)) + if (isSatisfiedByValue(value, item, valueType, Operator.EQ)) return true; } return false; } else { - Cell cell = current.getColumn(nameType.create(rowPrefix, column)); - ByteBuffer mapElementValue = null; - if (cell != null && cell.isLive(now)) - mapElementValue = mapType.getSerializer().getSerializedValue(cell.value(), collectionElement, mapType.getKeysType()); + Cell cell = getCell(row, column); + ByteBuffer mapElementValue = cell == null + ? null + : mapType.getSerializer().getSerializedValue(cell.value(), collectionElement, mapType.getKeysType()); for (ByteBuffer value : inValues) { if (value == null) @@ -433,9 +407,7 @@ public class ColumnCondition AbstractType elementsType = listType.getElementsType(); if (column.type.isMultiCell()) { - ByteBuffer columnValue = ElementAccessBound.getListItem( - collectionColumns(nameType.create(rowPrefix, column), current, now), - ElementAccessBound.getListIndex(collectionElement)); + ByteBuffer columnValue = ElementAccessBound.getListItem(getCells(row, column), ElementAccessBound.getListIndex(collectionElement)); for (ByteBuffer value : inValues) { @@ -445,10 +417,10 @@ public class ColumnCondition } else { - Cell cell = current.getColumn(nameType.create(rowPrefix, column)); - ByteBuffer listElementValue = null; - if (cell != null && cell.isLive(now)) - listElementValue = listType.getSerializer().getElement(cell.value(), ElementAccessBound.getListIndex(collectionElement)); + Cell cell = getCell(row, column); + ByteBuffer listElementValue = cell == null + ? null + : listType.getSerializer().getElement(cell.value(), ElementAccessBound.getListIndex(collectionElement)); for (ByteBuffer value : inValues) { @@ -479,13 +451,13 @@ public class ColumnCondition this.value = condition.value.bind(options); } - public boolean appliesTo(Composite rowPrefix, ColumnFamily current, final long now) throws InvalidRequestException + public boolean appliesTo(Row row) throws InvalidRequestException { CollectionType type = (CollectionType)column.type; if (type.isMultiCell()) { - Iterator iter = collectionColumns(current.metadata().comparator.create(rowPrefix, column), current, now); + Iterator iter = getCells(row, column); if (value == null) { if (operator == Operator.EQ) @@ -500,13 +472,13 @@ public class ColumnCondition } // frozen collections - Cell cell = current.getColumn(current.metadata().comparator.create(rowPrefix, column)); + Cell cell = getCell(row, column); if (value == null) { if (operator == Operator.EQ) - return cell == null || !cell.isLive(now); + return cell == null; else if (operator == Operator.NEQ) - return cell != null && cell.isLive(now); + return cell != null; else throw new InvalidRequestException(String.format("Invalid comparison with null for operator \"%s\"", operator)); } @@ -551,7 +523,7 @@ public class ColumnCondition return (operator == Operator.GT) || (operator == Operator.GTE) || (operator == Operator.NEQ); // for lists we use the cell value; for sets we use the cell name - ByteBuffer cellValue = isSet? iter.next().name().collectionElement() : iter.next().value(); + ByteBuffer cellValue = isSet ? iter.next().path().get(0) : iter.next().value(); int comparison = type.compare(cellValue, conditionIter.next()); if (comparison != 0) return evaluateComparisonWithOperator(comparison, operator); @@ -609,7 +581,7 @@ public class ColumnCondition Cell c = iter.next(); // compare the keys - int comparison = type.getKeysType().compare(c.name().collectionElement(), conditionEntry.getKey()); + int comparison = type.getKeysType().compare(c.path().get(0), conditionEntry.getKey()); if (comparison != 0) return evaluateComparisonWithOperator(comparison, operator); @@ -683,29 +655,27 @@ public class ColumnCondition } } - public boolean appliesTo(Composite rowPrefix, ColumnFamily current, final long now) throws InvalidRequestException + public boolean appliesTo(Row row) throws InvalidRequestException { CollectionType type = (CollectionType)column.type; - CellName name = current.metadata().comparator.create(rowPrefix, column); if (type.isMultiCell()) { // copy iterator contents so that we can properly reuse them for each comparison with an IN value - List cells = newArrayList(collectionColumns(name, current, now)); for (Term.Terminal value : inValues) { - if (CollectionBound.valueAppliesTo(type, cells.iterator(), value, Operator.EQ)) + if (CollectionBound.valueAppliesTo(type, getCells(row, column), value, Operator.EQ)) return true; } return false; } else { - Cell cell = current.getColumn(name); + Cell cell = getCell(row, column); for (Term.Terminal value : inValues) { if (value == null) { - if (cell == null || !cell.isLive(now)) + if (cell == null) return true; } else if (type.compare(value.get(Server.VERSION_3), cell.value()) == 0) diff --git a/src/java/org/apache/cassandra/cql3/ColumnIdentifier.java b/src/java/org/apache/cassandra/cql3/ColumnIdentifier.java index 823af94d3c..eafcf8d4db 100644 --- a/src/java/org/apache/cassandra/cql3/ColumnIdentifier.java +++ b/src/java/org/apache/cassandra/cql3/ColumnIdentifier.java @@ -20,6 +20,10 @@ package org.apache.cassandra.cql3; import java.nio.ByteBuffer; import java.util.List; import java.util.Locale; +import java.nio.ByteBuffer; +import java.util.concurrent.ConcurrentMap; + +import com.google.common.collect.MapMaker; import org.apache.cassandra.cache.IMeasurableMemory; import org.apache.cassandra.config.CFMetaData; @@ -28,7 +32,7 @@ import org.apache.cassandra.cql3.selection.Selectable; import org.apache.cassandra.cql3.selection.Selector; import org.apache.cassandra.cql3.selection.SimpleSelector; import org.apache.cassandra.db.marshal.AbstractType; -import org.apache.cassandra.db.marshal.CompositeType; +import org.apache.cassandra.exceptions.InvalidRequestException; import org.apache.cassandra.db.marshal.UTF8Type; import org.apache.cassandra.exceptions.InvalidRequestException; import org.apache.cassandra.utils.ByteBufferUtil; @@ -43,25 +47,57 @@ public class ColumnIdentifier extends org.apache.cassandra.cql3.selection.Select { public final ByteBuffer bytes; private final String text; + private final boolean interned; - private static final long EMPTY_SIZE = ObjectSizes.measure(new ColumnIdentifier("", true)); + private static final long EMPTY_SIZE = ObjectSizes.measure(new ColumnIdentifier(ByteBufferUtil.EMPTY_BYTE_BUFFER, "", false)); + + private static final ConcurrentMap internedInstances = new MapMaker().weakValues().makeMap(); public ColumnIdentifier(String rawText, boolean keepCase) { this.text = keepCase ? rawText : rawText.toLowerCase(Locale.US); this.bytes = ByteBufferUtil.bytes(this.text); + this.interned = false; } public ColumnIdentifier(ByteBuffer bytes, AbstractType type) { - this.bytes = bytes; - this.text = type.getString(bytes); + this(bytes, type.getString(bytes), false); } - public ColumnIdentifier(ByteBuffer bytes, String text) + private ColumnIdentifier(ByteBuffer bytes, String text, boolean interned) { this.bytes = bytes; this.text = text; + this.interned = interned; + } + + public static ColumnIdentifier getInterned(ByteBuffer bytes, AbstractType type) + { + return getInterned(bytes, type.getString(bytes)); + } + + public static ColumnIdentifier getInterned(String rawText, boolean keepCase) + { + String text = keepCase ? rawText : rawText.toLowerCase(Locale.US); + ByteBuffer bytes = ByteBufferUtil.bytes(text); + return getInterned(bytes, text); + } + + public static ColumnIdentifier getInterned(ByteBuffer bytes, String text) + { + ColumnIdentifier id = internedInstances.get(bytes); + if (id != null) + return id; + + ColumnIdentifier created = new ColumnIdentifier(bytes, text, true); + ColumnIdentifier previous = internedInstances.putIfAbsent(bytes, created); + return previous == null ? created : previous; + } + + public boolean isInterned() + { + return interned; } @Override @@ -73,8 +109,6 @@ public class ColumnIdentifier extends org.apache.cassandra.cql3.selection.Select @Override public final boolean equals(Object o) { - // Note: it's worth checking for reference equality since we intern those - // in SparseCellNameType if (this == o) return true; @@ -106,7 +140,7 @@ public class ColumnIdentifier extends org.apache.cassandra.cql3.selection.Select public ColumnIdentifier clone(AbstractAllocator allocator) { - return new ColumnIdentifier(allocator.clone(bytes), text); + return interned ? this : new ColumnIdentifier(allocator.clone(bytes), text, false); } public Selector.Factory newSelectorFactory(CFMetaData cfm, List defs) throws InvalidRequestException @@ -137,20 +171,22 @@ public class ColumnIdentifier extends org.apache.cassandra.cql3.selection.Select public ColumnIdentifier prepare(CFMetaData cfm) { - AbstractType comparator = cfm.comparator.asAbstractType(); - if (cfm.getIsDense() || comparator instanceof CompositeType || comparator instanceof UTF8Type) - return new ColumnIdentifier(text, true); + if (!cfm.isStaticCompactTable()) + return getInterned(text, true); - // We have a Thrift-created table with a non-text comparator. We need to parse column names with the comparator - // to get the correct ByteBuffer representation. However, this doesn't apply to key aliases, so we need to - // make a special check for those and treat them normally. See CASSANDRA-8178. + AbstractType thriftColumnNameType = cfm.thriftColumnNameType(); + if (thriftColumnNameType instanceof UTF8Type) + return getInterned(text, true); + + // We have a Thrift-created table with a non-text comparator. Check if we have a match column, otherwise assume we should use + // thriftColumnNameType ByteBuffer bufferName = ByteBufferUtil.bytes(text); - for (ColumnDefinition def : cfm.partitionKeyColumns()) + for (ColumnDefinition def : cfm.allColumns()) { if (def.name.bytes.equals(bufferName)) - return new ColumnIdentifier(text, true); + return def.name; } - return new ColumnIdentifier(comparator.fromString(rawText), text); + return getInterned(thriftColumnNameType.fromString(rawText), text); } public boolean processesSelection() diff --git a/src/java/org/apache/cassandra/cql3/Constants.java b/src/java/org/apache/cassandra/cql3/Constants.java index 07b848c3bb..859b1b58e5 100644 --- a/src/java/org/apache/cassandra/cql3/Constants.java +++ b/src/java/org/apache/cassandra/cql3/Constants.java @@ -23,8 +23,7 @@ import org.slf4j.Logger; import org.slf4j.LoggerFactory; import org.apache.cassandra.config.ColumnDefinition; import org.apache.cassandra.db.*; -import org.apache.cassandra.db.composites.CellName; -import org.apache.cassandra.db.composites.Composite; +import org.apache.cassandra.db.rows.*; import org.apache.cassandra.db.marshal.AbstractType; import org.apache.cassandra.db.marshal.BytesType; import org.apache.cassandra.db.marshal.CounterColumnType; @@ -319,14 +318,13 @@ public abstract class Constants super(column, t); } - public void execute(ByteBuffer rowKey, ColumnFamily cf, Composite prefix, UpdateParameters params) throws InvalidRequestException + public void execute(DecoratedKey partitionKey, Clustering clustering, Row.Writer writer, UpdateParameters params) throws InvalidRequestException { ByteBuffer value = t.bindAndGet(params.options); - if (value != ByteBufferUtil.UNSET_BYTE_BUFFER) // use reference equality and not object equality - { - CellName cname = cf.getComparator().create(prefix, column); - cf.addColumn(value == null ? params.makeTombstone(cname) : params.makeColumn(cname, value)); - } + if (value == null) + params.addTombstone(column, writer); + else if (value != ByteBufferUtil.UNSET_BYTE_BUFFER) // use reference equality and not object equality + params.addCell(clustering, column, writer, value); } } @@ -337,7 +335,7 @@ public abstract class Constants super(column, t); } - public void execute(ByteBuffer rowKey, ColumnFamily cf, Composite prefix, UpdateParameters params) throws InvalidRequestException + public void execute(DecoratedKey partitionKey, Clustering clustering, Row.Writer writer, UpdateParameters params) throws InvalidRequestException { ByteBuffer bytes = t.bindAndGet(params.options); if (bytes == null) @@ -346,8 +344,7 @@ public abstract class Constants return; long increment = ByteBufferUtil.toLong(bytes); - CellName cname = cf.getComparator().create(prefix, column); - cf.addColumn(params.makeCounter(cname, increment)); + params.addCounter(column, writer, increment); } } @@ -358,7 +355,7 @@ public abstract class Constants super(column, t); } - public void execute(ByteBuffer rowKey, ColumnFamily cf, Composite prefix, UpdateParameters params) throws InvalidRequestException + public void execute(DecoratedKey partitionKey, Clustering clustering, Row.Writer writer, UpdateParameters params) throws InvalidRequestException { ByteBuffer bytes = t.bindAndGet(params.options); if (bytes == null) @@ -370,8 +367,7 @@ public abstract class Constants if (increment == Long.MIN_VALUE) throw new InvalidRequestException("The negation of " + increment + " overflows supported counter precision (signed 8 bytes integer)"); - CellName cname = cf.getComparator().create(prefix, column); - cf.addColumn(params.makeCounter(cname, -increment)); + params.addCounter(column, writer, -increment); } } @@ -384,13 +380,12 @@ public abstract class Constants super(column, null); } - public void execute(ByteBuffer rowKey, ColumnFamily cf, Composite prefix, UpdateParameters params) throws InvalidRequestException + public void execute(DecoratedKey partitionKey, Clustering clustering, Row.Writer writer, UpdateParameters params) throws InvalidRequestException { - CellName cname = cf.getComparator().create(prefix, column); if (column.type.isMultiCell()) - cf.addAtom(params.makeRangeTombstone(cname.slice())); + params.setComplexDeletionTime(column, writer); else - cf.addColumn(params.makeTombstone(cname)); + params.addTombstone(column, writer); } }; } diff --git a/src/java/org/apache/cassandra/cql3/Cql.g b/src/java/org/apache/cassandra/cql3/Cql.g index 094b72ec8a..093a47ce5b 100644 --- a/src/java/org/apache/cassandra/cql3/Cql.g +++ b/src/java/org/apache/cassandra/cql3/Cql.g @@ -318,7 +318,7 @@ selectClause returns [List expr] selector returns [RawSelector s] @init{ ColumnIdentifier alias = null; } - : us=unaliasedSelector (K_AS c=ident { alias = c; })? { $s = new RawSelector(us, alias); } + : us=unaliasedSelector (K_AS c=noncol_ident { alias = c; })? { $s = new RawSelector(us, alias); } ; unaliasedSelector returns [Selectable.Raw s] @@ -339,7 +339,7 @@ selectionFunctionArgs returns [List a] selectCountClause returns [List expr] @init{ ColumnIdentifier alias = new ColumnIdentifier("count", false); } - : K_COUNT '(' countArgument ')' (K_AS c=ident { alias = c; })? { $expr = new ArrayList(); $expr.add( new RawSelector(new Selectable.WithFunction.Raw(FunctionName.nativeFunction("countRows"), Collections.emptyList()), alias));} + : K_COUNT '(' countArgument ')' (K_AS c=noncol_ident { alias = c; })? { $expr = new ArrayList(); $expr.add( new RawSelector(new Selectable.WithFunction.Raw(FunctionName.nativeFunction("countRows"), Collections.emptyList()), alias));} ; countArgument @@ -404,7 +404,7 @@ jsonInsertStatement [CFName cf] returns [UpdateStatement.ParsedInsertJson expr] jsonValue returns [Json.Raw value] : | s=STRING_LITERAL { $value = new Json.Literal($s.text); } - | ':' id=ident { $value = newJsonBindVariables(id); } + | ':' id=noncol_ident { $value = newJsonBindVariables(id); } | QMARK { $value = newJsonBindVariables(null); } ; @@ -604,8 +604,8 @@ createFunctionStatement returns [CreateFunctionStatement expr] fn=functionName '(' ( - k=ident v=comparatorType { argsNames.add(k); argsTypes.add(v); } - ( ',' k=ident v=comparatorType { argsNames.add(k); argsTypes.add(v); } )* + k=noncol_ident v=comparatorType { argsNames.add(k); argsTypes.add(v); } + ( ',' k=noncol_ident v=comparatorType { argsNames.add(k); argsTypes.add(v); } )* )? ')' ( (K_RETURNS K_NULL) | (K_CALLED { calledOnNullInput=true; })) K_ON K_NULL K_INPUT @@ -706,7 +706,7 @@ createTypeStatement returns [CreateTypeStatement expr] ; typeColumns[CreateTypeStatement expr] - : k=ident v=comparatorType { $expr.addDefinition(k, v); } + : k=noncol_ident v=comparatorType { $expr.addDefinition(k, v); } ; @@ -801,12 +801,12 @@ alterTableStatement returns [AlterTableStatement expr] */ alterTypeStatement returns [AlterTypeStatement expr] : K_ALTER K_TYPE name=userTypeName - ( K_ALTER f=ident K_TYPE v=comparatorType { $expr = AlterTypeStatement.alter(name, f, v); } - | K_ADD f=ident v=comparatorType { $expr = AlterTypeStatement.addition(name, f, v); } + ( K_ALTER f=noncol_ident K_TYPE v=comparatorType { $expr = AlterTypeStatement.alter(name, f, v); } + | K_ADD f=noncol_ident v=comparatorType { $expr = AlterTypeStatement.addition(name, f, v); } | K_RENAME { Map renames = new HashMap(); } - id1=ident K_TO toId1=ident { renames.put(id1, toId1); } - ( K_AND idn=ident K_TO toIdn=ident { renames.put(idn, toIdn); } )* + id1=noncol_ident K_TO toId1=noncol_ident { renames.put(id1, toId1); } + ( K_AND idn=noncol_ident K_TO toIdn=noncol_ident { renames.put(idn, toIdn); } )* { $expr = AlterTypeStatement.renames(name, renames); } ) ; @@ -1112,8 +1112,15 @@ cident returns [ColumnIdentifier.Raw id] | k=unreserved_keyword { $id = new ColumnIdentifier.Raw(k, false); } ; -// Identifiers that do not refer to columns or where the comparator is known to be text +// Column identifiers where the comparator is known to be text ident returns [ColumnIdentifier id] + : t=IDENT { $id = ColumnIdentifier.getInterned($t.text, false); } + | t=QUOTED_NAME { $id = ColumnIdentifier.getInterned($t.text, true); } + | k=unreserved_keyword { $id = ColumnIdentifier.getInterned(k, false); } + ; + +// Identifiers that do not refer to columns +noncol_ident returns [ColumnIdentifier id] : t=IDENT { $id = new ColumnIdentifier($t.text, false); } | t=QUOTED_NAME { $id = new ColumnIdentifier($t.text, true); } | k=unreserved_keyword { $id = new ColumnIdentifier(k, false); } @@ -1136,7 +1143,7 @@ columnFamilyName returns [CFName name] ; userTypeName returns [UTName name] - : (ks=ident '.')? ut=non_type_ident { return new UTName(ks, ut); } + : (ks=noncol_ident '.')? ut=non_type_ident { return new UTName(ks, ut); } ; userOrRoleName returns [RoleName name] @@ -1211,7 +1218,7 @@ usertypeLiteral returns [UserTypes.Literal ut] @init{ Map m = new HashMap(); } @after{ $ut = new UserTypes.Literal(m); } // We don't allow empty literals because that conflicts with sets/maps and is currently useless since we don't allow empty user types - : '{' k1=ident ':' v1=term { m.put(k1, v1); } ( ',' kn=ident ':' vn=term { m.put(kn, vn); } )* '}' + : '{' k1=noncol_ident ':' v1=term { m.put(k1, v1); } ( ',' kn=noncol_ident ':' vn=term { m.put(kn, vn); } )* '}' ; tupleLiteral returns [Tuples.Literal tt] @@ -1226,14 +1233,14 @@ value returns [Term.Raw value] | u=usertypeLiteral { $value = u; } | t=tupleLiteral { $value = t; } | K_NULL { $value = Constants.NULL_LITERAL; } - | ':' id=ident { $value = newBindVariables(id); } + | ':' id=noncol_ident { $value = newBindVariables(id); } | QMARK { $value = newBindVariables(null); } ; intValue returns [Term.Raw value] : | t=INTEGER { $value = Constants.Literal.integer($t.text); } - | ':' id=ident { $value = newBindVariables(id); } + | ':' id=noncol_ident { $value = newBindVariables(id); } | QMARK { $value = newBindVariables(null); } ; @@ -1334,8 +1341,8 @@ properties[PropertyDefinitions props] ; property[PropertyDefinitions props] - : k=ident '=' simple=propertyValue { try { $props.addProperty(k.toString(), simple); } catch (SyntaxException e) { addRecognitionError(e.getMessage()); } } - | k=ident '=' map=mapLiteral { try { $props.addProperty(k.toString(), convertPropertyMap(map)); } catch (SyntaxException e) { addRecognitionError(e.getMessage()); } } + : k=noncol_ident '=' simple=propertyValue { try { $props.addProperty(k.toString(), simple); } catch (SyntaxException e) { addRecognitionError(e.getMessage()); } } + | k=noncol_ident '=' map=mapLiteral { try { $props.addProperty(k.toString(), convertPropertyMap(map)); } catch (SyntaxException e) { addRecognitionError(e.getMessage()); } } ; propertyValue returns [String str] @@ -1388,7 +1395,7 @@ relation[List clauses] inMarker returns [AbstractMarker.INRaw marker] : QMARK { $marker = newINBindVariables(null); } - | ':' name=ident { $marker = newINBindVariables(name); } + | ':' name=noncol_ident { $marker = newINBindVariables(name); } ; tupleOfIdentifiers returns [List ids] @@ -1408,7 +1415,7 @@ tupleOfTupleLiterals returns [List literals] markerForTuple returns [Tuples.Raw marker] : QMARK { $marker = newTupleBindVariables(null); } - | ':' name=ident { $marker = newTupleBindVariables(name); } + | ':' name=noncol_ident { $marker = newTupleBindVariables(name); } ; tupleOfMarkersForTuples returns [List markers] @@ -1418,7 +1425,7 @@ tupleOfMarkersForTuples returns [List markers] inMarkerForTuple returns [Tuples.INRaw marker] : QMARK { $marker = newTupleINBindVariables(null); } - | ':' name=ident { $marker = newTupleINBindVariables(name); } + | ':' name=noncol_ident { $marker = newTupleINBindVariables(name); } ; comparatorType returns [CQL3Type.Raw t] diff --git a/src/java/org/apache/cassandra/cql3/Lists.java b/src/java/org/apache/cassandra/cql3/Lists.java index da8c48a2c7..5da2b37daf 100644 --- a/src/java/org/apache/cassandra/cql3/Lists.java +++ b/src/java/org/apache/cassandra/cql3/Lists.java @@ -21,15 +21,16 @@ import static org.apache.cassandra.cql3.Constants.UNSET_VALUE; import java.nio.ByteBuffer; import java.util.ArrayList; +import java.util.Iterator; import java.util.List; import java.util.concurrent.atomic.AtomicReference; +import com.google.common.collect.Iterators; + import org.apache.cassandra.config.ColumnDefinition; import org.apache.cassandra.cql3.functions.Function; -import org.apache.cassandra.db.Cell; -import org.apache.cassandra.db.ColumnFamily; -import org.apache.cassandra.db.composites.CellName; -import org.apache.cassandra.db.composites.Composite; +import org.apache.cassandra.db.*; +import org.apache.cassandra.db.rows.*; import org.apache.cassandra.db.marshal.Int32Type; import org.apache.cassandra.db.marshal.ListType; import org.apache.cassandra.exceptions.InvalidRequestException; @@ -299,20 +300,37 @@ public abstract class Lists super(column, t); } - public void execute(ByteBuffer rowKey, ColumnFamily cf, Composite prefix, UpdateParameters params) throws InvalidRequestException + public void execute(DecoratedKey partitionKey, Clustering clustering, Row.Writer writer, UpdateParameters params) throws InvalidRequestException { Term.Terminal value = t.bind(params.options); - if (column.type.isMultiCell() && value != UNSET_VALUE) - { - // delete + append - CellName name = cf.getComparator().create(prefix, column); - cf.addAtom(params.makeTombstoneForOverwrite(name.slice())); - } - if (value != UNSET_VALUE) - Appender.doAppend(cf, prefix, column, params, value); + if (value == UNSET_VALUE) + return; + + // delete + append + if (column.type.isMultiCell()) + params.setComplexDeletionTimeForOverwrite(column, writer); + Appender.doAppend(value, clustering, writer, column, params); } } + private static int existingSize(Row row, ColumnDefinition column) + { + if (row == null) + return 0; + + Iterator cells = row.getCells(column); + return cells == null ? 0 : Iterators.size(cells); + } + + private static Cell existingElement(Row row, ColumnDefinition column, int idx) + { + assert row != null; + Iterator cells = row.getCells(column); + assert cells != null; + + return Iterators.get(cells, idx); + } + public static class SetterByIndex extends Operation { private final Term idx; @@ -336,7 +354,7 @@ public abstract class Lists idx.collectMarkerSpecification(boundNames); } - public void execute(ByteBuffer rowKey, ColumnFamily cf, Composite prefix, UpdateParameters params) throws InvalidRequestException + public void execute(DecoratedKey partitionKey, Clustering clustering, Row.Writer writer, UpdateParameters params) throws InvalidRequestException { // we should not get here for frozen lists assert column.type.isMultiCell() : "Attempted to set an individual element on a frozen list"; @@ -349,17 +367,18 @@ public abstract class Lists if (index == ByteBufferUtil.UNSET_BYTE_BUFFER) throw new InvalidRequestException("Invalid unset value for list index"); - List existingList = params.getPrefetchedList(rowKey, column.name); + Row existingRow = params.getPrefetchedRow(partitionKey, clustering); + int existingSize = existingSize(existingRow, column); int idx = ByteBufferUtil.toInt(index); - if (existingList == null || existingList.size() == 0) + if (existingSize == 0) throw new InvalidRequestException("Attempted to set an element on a list which is null"); - if (idx < 0 || idx >= existingList.size()) - throw new InvalidRequestException(String.format("List index %d out of bound, list has size %d", idx, existingList.size())); + if (idx < 0 || idx >= existingSize) + throw new InvalidRequestException(String.format("List index %d out of bound, list has size %d", idx, existingSize)); - CellName elementName = existingList.get(idx).name(); + CellPath elementPath = existingElement(existingRow, column, idx).path(); if (value == null) { - cf.addColumn(params.makeTombstone(elementName)); + params.addTombstone(column, writer); } else if (value != ByteBufferUtil.UNSET_BYTE_BUFFER) { @@ -369,7 +388,7 @@ public abstract class Lists FBUtilities.MAX_UNSIGNED_SHORT, value.remaining())); - cf.addColumn(params.makeColumn(elementName, value)); + params.addCell(clustering, column, writer, elementPath, value); } } } @@ -381,15 +400,14 @@ public abstract class Lists super(column, t); } - public void execute(ByteBuffer rowKey, ColumnFamily cf, Composite prefix, UpdateParameters params) throws InvalidRequestException + public void execute(DecoratedKey partitionKey, Clustering clustering, Row.Writer writer, UpdateParameters params) throws InvalidRequestException { assert column.type.isMultiCell() : "Attempted to append to a frozen list"; Term.Terminal value = t.bind(params.options); - if (value != UNSET_VALUE) - doAppend(cf, prefix, column, params, value); + doAppend(value, clustering, writer, column, params); } - static void doAppend(ColumnFamily cf, Composite prefix, ColumnDefinition column, UpdateParameters params, Term.Terminal value) throws InvalidRequestException + static void doAppend(Term.Terminal value, Clustering clustering, Row.Writer writer, ColumnDefinition column, UpdateParameters params) throws InvalidRequestException { if (column.type.isMultiCell()) { @@ -401,17 +419,16 @@ public abstract class Lists for (ByteBuffer buffer : ((Value) value).elements) { ByteBuffer uuid = ByteBuffer.wrap(UUIDGen.getTimeUUIDBytes()); - cf.addColumn(params.makeColumn(cf.getComparator().create(prefix, column, uuid), buffer)); + params.addCell(clustering, column, writer, CellPath.create(uuid), buffer); } } else { // for frozen lists, we're overwriting the whole cell value - CellName name = cf.getComparator().create(prefix, column); if (value == null) - cf.addAtom(params.makeTombstone(name)); + params.addTombstone(column, writer); else - cf.addColumn(params.makeColumn(name, value.get(Server.CURRENT_VERSION))); + params.addCell(clustering, column, writer, value.get(Server.CURRENT_VERSION)); } } } @@ -423,7 +440,7 @@ public abstract class Lists super(column, t); } - public void execute(ByteBuffer rowKey, ColumnFamily cf, Composite prefix, UpdateParameters params) throws InvalidRequestException + public void execute(DecoratedKey partitionKey, Clustering clustering, Row.Writer writer, UpdateParameters params) throws InvalidRequestException { assert column.type.isMultiCell() : "Attempted to prepend to a frozen list"; Term.Terminal value = t.bind(params.options); @@ -437,7 +454,7 @@ public abstract class Lists { PrecisionTime pt = PrecisionTime.getNext(time); ByteBuffer uuid = ByteBuffer.wrap(UUIDGen.getTimeUUIDBytes(pt.millis, pt.nanos)); - cf.addColumn(params.makeColumn(cf.getComparator().create(prefix, column, uuid), toAdd.get(i))); + params.addCell(clustering, column, writer, CellPath.create(uuid), toAdd.get(i)); } } } @@ -455,30 +472,28 @@ public abstract class Lists return true; } - public void execute(ByteBuffer rowKey, ColumnFamily cf, Composite prefix, UpdateParameters params) throws InvalidRequestException + public void execute(DecoratedKey partitionKey, Clustering clustering, Row.Writer writer, UpdateParameters params) throws InvalidRequestException { assert column.type.isMultiCell() : "Attempted to delete from a frozen list"; - List existingList = params.getPrefetchedList(rowKey, column.name); + // We want to call bind before possibly returning to reject queries where the value provided is not a list. Term.Terminal value = t.bind(params.options); - if (existingList == null) - throw new InvalidRequestException("Attempted to delete an element from a list which is null"); - if (existingList.isEmpty()) - return; - - if (value == null || value == UNSET_VALUE) + Row existingRow = params.getPrefetchedRow(partitionKey, clustering); + Iterator cells = existingRow == null ? null : existingRow.getCells(column); + if (value == null || value == UNSET_VALUE || cells == null) return; // Note: below, we will call 'contains' on this toDiscard list for each element of existingList. // Meaning that if toDiscard is big, converting it to a HashSet might be more efficient. However, // the read-before-write this operation requires limits its usefulness on big lists, so in practice // toDiscard will be small and keeping a list will be more efficient. - List toDiscard = ((Value) value).elements; - for (Cell cell : existingList) + List toDiscard = ((Value)value).elements; + while (cells.hasNext()) { + Cell cell = cells.next(); if (toDiscard.contains(cell.value())) - cf.addColumn(params.makeTombstone(cell.name())); + params.addTombstone(column, writer, cell.path()); } } } @@ -496,7 +511,7 @@ public abstract class Lists return true; } - public void execute(ByteBuffer rowKey, ColumnFamily cf, Composite prefix, UpdateParameters params) throws InvalidRequestException + public void execute(DecoratedKey partitionKey, Clustering clustering, Row.Writer writer, UpdateParameters params) throws InvalidRequestException { assert column.type.isMultiCell() : "Attempted to delete an item by index from a frozen list"; Term.Terminal index = t.bind(params.options); @@ -505,15 +520,15 @@ public abstract class Lists if (index == Constants.UNSET_VALUE) return; - List existingList = params.getPrefetchedList(rowKey, column.name); + Row existingRow = params.getPrefetchedRow(partitionKey, clustering); + int existingSize = existingSize(existingRow, column); int idx = ByteBufferUtil.toInt(index.get(params.options.getProtocolVersion())); - if (existingList == null || existingList.size() == 0) + if (existingSize == 0) throw new InvalidRequestException("Attempted to delete an element from a list which is null"); - if (idx < 0 || idx >= existingList.size()) - throw new InvalidRequestException(String.format("List index %d out of bound, list has size %d", idx, existingList.size())); + if (idx < 0 || idx >= existingSize) + throw new InvalidRequestException(String.format("List index %d out of bound, list has size %d", idx, existingSize)); - CellName elementName = existingList.get(idx).name(); - cf.addColumn(params.makeTombstone(elementName)); + params.addTombstone(column, writer, existingElement(existingRow, column, idx).path()); } } } diff --git a/src/java/org/apache/cassandra/cql3/Maps.java b/src/java/org/apache/cassandra/cql3/Maps.java index 5bb3a48f1e..2644108cf9 100644 --- a/src/java/org/apache/cassandra/cql3/Maps.java +++ b/src/java/org/apache/cassandra/cql3/Maps.java @@ -26,9 +26,9 @@ import com.google.common.collect.Iterables; import org.apache.cassandra.config.ColumnDefinition; import org.apache.cassandra.cql3.functions.Function; -import org.apache.cassandra.db.ColumnFamily; -import org.apache.cassandra.db.composites.CellName; -import org.apache.cassandra.db.composites.Composite; +import org.apache.cassandra.db.Clustering; +import org.apache.cassandra.db.DecoratedKey; +import org.apache.cassandra.db.rows.*; import org.apache.cassandra.db.marshal.MapType; import org.apache.cassandra.exceptions.InvalidRequestException; import org.apache.cassandra.serializers.CollectionSerializer; @@ -290,17 +290,16 @@ public abstract class Maps super(column, t); } - public void execute(ByteBuffer rowKey, ColumnFamily cf, Composite prefix, UpdateParameters params) throws InvalidRequestException + public void execute(DecoratedKey partitionKey, Clustering clustering, Row.Writer writer, UpdateParameters params) throws InvalidRequestException { Term.Terminal value = t.bind(params.options); - if (column.type.isMultiCell() && value != UNSET_VALUE) - { - // delete + put - CellName name = cf.getComparator().create(prefix, column); - cf.addAtom(params.makeTombstoneForOverwrite(name.slice())); - } - if (value != UNSET_VALUE) - Putter.doPut(cf, prefix, column, params, value); + if (value == UNSET_VALUE) + return; + + // delete + put + if (column.type.isMultiCell()) + params.setComplexDeletionTimeForOverwrite(column, writer); + Putter.doPut(value, clustering, writer, column, params); } } @@ -321,7 +320,7 @@ public abstract class Maps k.collectMarkerSpecification(boundNames); } - public void execute(ByteBuffer rowKey, ColumnFamily cf, Composite prefix, UpdateParameters params) throws InvalidRequestException + public void execute(DecoratedKey partitionKey, Clustering clustering, Row.Writer writer, UpdateParameters params) throws InvalidRequestException { assert column.type.isMultiCell() : "Attempted to set a value for a single key on a frozen map"; ByteBuffer key = k.bindAndGet(params.options); @@ -331,11 +330,11 @@ public abstract class Maps if (key == ByteBufferUtil.UNSET_BYTE_BUFFER) throw new InvalidRequestException("Invalid unset map key"); - CellName cellName = cf.getComparator().create(prefix, column, key); + CellPath path = CellPath.create(key); if (value == null) { - cf.addColumn(params.makeTombstone(cellName)); + params.addTombstone(column, writer, path); } else if (value != ByteBufferUtil.UNSET_BYTE_BUFFER) { @@ -345,7 +344,7 @@ public abstract class Maps FBUtilities.MAX_UNSIGNED_SHORT, value.remaining())); - cf.addColumn(params.makeColumn(cellName, value)); + params.addCell(clustering, column, writer, path, value); } } } @@ -357,15 +356,15 @@ public abstract class Maps super(column, t); } - public void execute(ByteBuffer rowKey, ColumnFamily cf, Composite prefix, UpdateParameters params) throws InvalidRequestException + public void execute(DecoratedKey partitionKey, Clustering clustering, Row.Writer writer, UpdateParameters params) throws InvalidRequestException { assert column.type.isMultiCell() : "Attempted to add items to a frozen map"; Term.Terminal value = t.bind(params.options); if (value != UNSET_VALUE) - doPut(cf, prefix, column, params, value); + doPut(value, clustering, writer, column, params); } - static void doPut(ColumnFamily cf, Composite prefix, ColumnDefinition column, UpdateParameters params, Term.Terminal value) throws InvalidRequestException + static void doPut(Term.Terminal value, Clustering clustering, Row.Writer writer, ColumnDefinition column, UpdateParameters params) throws InvalidRequestException { if (column.type.isMultiCell()) { @@ -374,19 +373,15 @@ public abstract class Maps Map elements = ((Value) value).map; for (Map.Entry entry : elements.entrySet()) - { - CellName cellName = cf.getComparator().create(prefix, column, entry.getKey()); - cf.addColumn(params.makeColumn(cellName, entry.getValue())); - } + params.addCell(clustering, column, writer, CellPath.create(entry.getKey()), entry.getValue()); } else { // for frozen maps, we're overwriting the whole cell - CellName cellName = cf.getComparator().create(prefix, column); if (value == null) - cf.addAtom(params.makeTombstone(cellName)); + params.addTombstone(column, writer); else - cf.addColumn(params.makeColumn(cellName, value.get(Server.CURRENT_VERSION))); + params.addCell(clustering, column, writer, value.get(Server.CURRENT_VERSION)); } } } @@ -398,7 +393,7 @@ public abstract class Maps super(column, k); } - public void execute(ByteBuffer rowKey, ColumnFamily cf, Composite prefix, UpdateParameters params) throws InvalidRequestException + public void execute(DecoratedKey partitionKey, Clustering clustering, Row.Writer writer, UpdateParameters params) throws InvalidRequestException { assert column.type.isMultiCell() : "Attempted to delete a single key in a frozen map"; Term.Terminal key = t.bind(params.options); @@ -407,8 +402,7 @@ public abstract class Maps if (key == Constants.UNSET_VALUE) throw new InvalidRequestException("Invalid unset map key"); - CellName cellName = cf.getComparator().create(prefix, column, key.get(params.options.getProtocolVersion())); - cf.addColumn(params.makeTombstone(cellName)); + params.addTombstone(column, writer, CellPath.create(key.get(params.options.getProtocolVersion()))); } } } diff --git a/src/java/org/apache/cassandra/cql3/MultiColumnRelation.java b/src/java/org/apache/cassandra/cql3/MultiColumnRelation.java index b54bdd058a..7735c574af 100644 --- a/src/java/org/apache/cassandra/cql3/MultiColumnRelation.java +++ b/src/java/org/apache/cassandra/cql3/MultiColumnRelation.java @@ -131,7 +131,7 @@ public class MultiColumnRelation extends Relation { List receivers = receivers(cfm); Term term = toTerm(receivers, getValue(), cfm.ksName, boundNames); - return new MultiColumnRestriction.EQ(receivers, term); + return new MultiColumnRestriction.EQRestriction(receivers, term); } @Override @@ -143,9 +143,9 @@ public class MultiColumnRelation extends Relation if (terms == null) { Term term = toTerm(receivers, getValue(), cfm.ksName, boundNames); - return new MultiColumnRestriction.InWithMarker(receivers, (AbstractMarker) term); + return new MultiColumnRestriction.InRestrictionWithMarker(receivers, (AbstractMarker) term); } - return new MultiColumnRestriction.InWithValues(receivers, terms); + return new MultiColumnRestriction.InRestrictionWithValues(receivers, terms); } @Override @@ -156,7 +156,7 @@ public class MultiColumnRelation extends Relation { List receivers = receivers(cfm); Term term = toTerm(receivers(cfm), getValue(), cfm.ksName, boundNames); - return new MultiColumnRestriction.Slice(receivers, bound, inclusive, term); + return new MultiColumnRestriction.SliceRestriction(receivers, bound, inclusive, term); } @Override @@ -214,4 +214,4 @@ public class MultiColumnRelation extends Relation .append(valuesOrMarker) .toString(); } -} \ No newline at end of file +} diff --git a/src/java/org/apache/cassandra/cql3/Operation.java b/src/java/org/apache/cassandra/cql3/Operation.java index 4701a96392..5e72e7f189 100644 --- a/src/java/org/apache/cassandra/cql3/Operation.java +++ b/src/java/org/apache/cassandra/cql3/Operation.java @@ -17,13 +17,13 @@ */ package org.apache.cassandra.cql3; -import java.nio.ByteBuffer; import java.util.Collections; import org.apache.cassandra.config.ColumnDefinition; import org.apache.cassandra.cql3.functions.Function; -import org.apache.cassandra.db.ColumnFamily; -import org.apache.cassandra.db.composites.Composite; +import org.apache.cassandra.db.Clustering; +import org.apache.cassandra.db.DecoratedKey; +import org.apache.cassandra.db.rows.Row; import org.apache.cassandra.db.marshal.*; import org.apache.cassandra.exceptions.InvalidRequestException; @@ -86,12 +86,12 @@ public abstract class Operation /** * Execute the operation. * - * @param rowKey row key for the update. - * @param cf the column family to which to add the updates generated by this operation. - * @param prefix the prefix that identify the CQL3 row this operation applies to. + * @param partitionKey partition key for the update. + * @param clustering the clustering for the row on which the operation applies + * @param writer the row update to which to add the updates generated by this operation. * @param params parameters of the update. */ - public abstract void execute(ByteBuffer rowKey, ColumnFamily cf, Composite prefix, UpdateParameters params) throws InvalidRequestException; + public abstract void execute(DecoratedKey partitionKey, Clustering clustering, Row.Writer writer, UpdateParameters params) throws InvalidRequestException; /** * A parsed raw UPDATE operation. diff --git a/src/java/org/apache/cassandra/cql3/Operator.java b/src/java/org/apache/cassandra/cql3/Operator.java index 86bcbd38e8..5ae988533f 100644 --- a/src/java/org/apache/cassandra/cql3/Operator.java +++ b/src/java/org/apache/cassandra/cql3/Operator.java @@ -20,6 +20,9 @@ package org.apache.cassandra.cql3; import java.io.DataInput; import java.io.DataOutput; import java.io.IOException; +import java.nio.ByteBuffer; + +import org.apache.cassandra.db.marshal.AbstractType; public enum Operator { @@ -38,12 +41,6 @@ public enum Operator { return "<"; } - - @Override - public Operator reverse() - { - return GT; - } }, LTE(3) { @@ -52,12 +49,6 @@ public enum Operator { return "<="; } - - @Override - public Operator reverse() - { - return GTE; - } }, GTE(1) { @@ -66,12 +57,6 @@ public enum Operator { return ">="; } - - @Override - public Operator reverse() - { - return LTE; - } }, GT(2) { @@ -80,12 +65,6 @@ public enum Operator { return ">"; } - - @Override - public Operator reverse() - { - return LT; - } }, IN(7) { @@ -152,19 +131,42 @@ public enum Operator throw new IOException(String.format("Cannot resolve Relation.Type from binary representation: %s", b)); } + /** + * Whether 2 values satisfy this operator (given the type they should be compared with). + * + * @throws AssertionError for IN, CONTAINS and CONTAINS_KEY as this doesn't make sense for this function. + */ + public boolean isSatisfiedBy(AbstractType type, ByteBuffer leftOperand, ByteBuffer rightOperand) + { + int comparison = type.compareForCQL(leftOperand, rightOperand); + switch (this) + { + case EQ: + return comparison == 0; + case LT: + return comparison < 0; + case LTE: + return comparison <= 0; + case GT: + return comparison > 0; + case GTE: + return comparison >= 0; + case NEQ: + return comparison != 0; + default: + // we shouldn't get IN, CONTAINS, or CONTAINS KEY here + throw new AssertionError(); + } + } + + public int serializedSize() + { + return 4; + } + @Override public String toString() { return this.name(); } - - /** - * Returns the reverse operator if this one. - * - * @return the reverse operator of this one. - */ - public Operator reverse() - { - return this; - } } diff --git a/src/java/org/apache/cassandra/cql3/QueryProcessor.java b/src/java/org/apache/cassandra/cql3/QueryProcessor.java index a071eb983a..afe2269cce 100644 --- a/src/java/org/apache/cassandra/cql3/QueryProcessor.java +++ b/src/java/org/apache/cassandra/cql3/QueryProcessor.java @@ -41,13 +41,14 @@ import org.apache.cassandra.cql3.functions.FunctionName; import org.apache.cassandra.cql3.functions.Functions; import org.apache.cassandra.cql3.statements.*; import org.apache.cassandra.db.*; -import org.apache.cassandra.db.composites.*; +import org.apache.cassandra.db.rows.RowIterator; +import org.apache.cassandra.db.partitions.PartitionIterator; +import org.apache.cassandra.db.partitions.PartitionIterators; import org.apache.cassandra.db.marshal.AbstractType; import org.apache.cassandra.exceptions.*; import org.apache.cassandra.metrics.CQLMetrics; import org.apache.cassandra.service.*; import org.apache.cassandra.service.pager.QueryPager; -import org.apache.cassandra.service.pager.QueryPagers; import org.apache.cassandra.thrift.ThriftClientState; import org.apache.cassandra.tracing.Tracing; import org.apache.cassandra.transport.messages.ResultMessage; @@ -192,28 +193,6 @@ public class QueryProcessor implements QueryHandler } } - public static void validateCellNames(Iterable cellNames, CellNameType type) throws InvalidRequestException - { - for (CellName name : cellNames) - validateCellName(name, type); - } - - public static void validateCellName(CellName name, CellNameType type) throws InvalidRequestException - { - validateComposite(name, type); - if (name.isEmpty()) - throw new InvalidRequestException("Invalid empty value for clustering column of COMPACT TABLE"); - } - - public static void validateComposite(Composite name, CType type) throws InvalidRequestException - { - long serializedSize = type.serializer().serializedSize(name, TypeSizes.NATIVE); - if (serializedSize > Cell.MAX_NAME_LENGTH) - throw new InvalidRequestException(String.format("The sum of all clustering columns is too long (%s > %s)", - serializedSize, - Cell.MAX_NAME_LENGTH)); - } - public ResultMessage processStatement(CQLStatement statement, QueryState queryState, QueryOptions options) throws RequestExecutionException, RequestValidationException { @@ -276,6 +255,11 @@ public class QueryProcessor implements QueryHandler } private static QueryOptions makeInternalOptions(ParsedStatement.Prepared prepared, Object[] values) + { + return makeInternalOptions(prepared, values, ConsistencyLevel.ONE); + } + + private static QueryOptions makeInternalOptions(ParsedStatement.Prepared prepared, Object[] values, ConsistencyLevel cl) { if (prepared.boundNames.size() != values.length) throw new IllegalArgumentException(String.format("Invalid number of values. Expecting %d but got %d", prepared.boundNames.size(), values.length)); @@ -287,7 +271,7 @@ public class QueryProcessor implements QueryHandler AbstractType type = prepared.boundNames.get(i).type; boundValues.add(value instanceof ByteBuffer || value == null ? (ByteBuffer)value : type.decompose(value)); } - return QueryOptions.forInternalCalls(boundValues); + return QueryOptions.forInternalCalls(cl, boundValues); } private static ParsedStatement.Prepared prepareInternal(String query) throws RequestValidationException @@ -313,6 +297,24 @@ public class QueryProcessor implements QueryHandler return null; } + public static UntypedResultSet execute(String query, ConsistencyLevel cl, QueryState state, Object... values) + throws RequestExecutionException + { + try + { + ParsedStatement.Prepared prepared = prepareInternal(query); + ResultMessage result = prepared.statement.execute(state, makeInternalOptions(prepared, values)); + if (result instanceof ResultMessage.Rows) + return UntypedResultSet.create(((ResultMessage.Rows)result).result); + else + return null; + } + catch (RequestValidationException e) + { + throw new RuntimeException("Error validating " + query, e); + } + } + public static UntypedResultSet executeInternalWithPaging(String query, int pageSize, Object... values) { ParsedStatement.Prepared prepared = prepareInternal(query); @@ -320,7 +322,7 @@ public class QueryProcessor implements QueryHandler throw new IllegalArgumentException("Only SELECTs can be paged"); SelectStatement select = (SelectStatement)prepared.statement; - QueryPager pager = QueryPagers.localPager(select.getPageableCommand(makeInternalOptions(prepared, values))); + QueryPager pager = select.getQuery(makeInternalOptions(prepared, values), FBUtilities.nowInSeconds()).getPager(null); return UntypedResultSet.create(select, pager, pageSize); } @@ -339,16 +341,19 @@ public class QueryProcessor implements QueryHandler return null; } - public static UntypedResultSet resultify(String query, Row row) + public static UntypedResultSet resultify(String query, RowIterator partition) { - return resultify(query, Collections.singletonList(row)); + return resultify(query, PartitionIterators.singletonIterator(partition)); } - public static UntypedResultSet resultify(String query, List rows) + public static UntypedResultSet resultify(String query, PartitionIterator partitions) { - SelectStatement ss = (SelectStatement) getStatement(query, null).statement; - ResultSet cqlRows = ss.process(rows); - return UntypedResultSet.create(cqlRows); + try (PartitionIterator iter = partitions) + { + SelectStatement ss = (SelectStatement) getStatement(query, null).statement; + ResultSet cqlRows = ss.process(iter, FBUtilities.nowInSeconds()); + return UntypedResultSet.create(cqlRows); + } } public ResultMessage.Prepared prepare(String query, diff --git a/src/java/org/apache/cassandra/cql3/Sets.java b/src/java/org/apache/cassandra/cql3/Sets.java index 093f1dc430..c03005d368 100644 --- a/src/java/org/apache/cassandra/cql3/Sets.java +++ b/src/java/org/apache/cassandra/cql3/Sets.java @@ -26,12 +26,10 @@ import com.google.common.base.Joiner; import org.apache.cassandra.config.ColumnDefinition; import org.apache.cassandra.cql3.functions.Function; -import org.apache.cassandra.db.ColumnFamily; -import org.apache.cassandra.db.composites.CellName; -import org.apache.cassandra.db.composites.Composite; -import org.apache.cassandra.db.marshal.AbstractType; -import org.apache.cassandra.db.marshal.MapType; -import org.apache.cassandra.db.marshal.SetType; +import org.apache.cassandra.db.Clustering; +import org.apache.cassandra.db.DecoratedKey; +import org.apache.cassandra.db.rows.*; +import org.apache.cassandra.db.marshal.*; import org.apache.cassandra.exceptions.InvalidRequestException; import org.apache.cassandra.serializers.CollectionSerializer; import org.apache.cassandra.serializers.MarshalException; @@ -259,17 +257,16 @@ public abstract class Sets super(column, t); } - public void execute(ByteBuffer rowKey, ColumnFamily cf, Composite prefix, UpdateParameters params) throws InvalidRequestException + public void execute(DecoratedKey partitionKey, Clustering clustering, Row.Writer writer, UpdateParameters params) throws InvalidRequestException { Term.Terminal value = t.bind(params.options); - if (column.type.isMultiCell() && value != UNSET_VALUE) - { - // delete + add - CellName name = cf.getComparator().create(prefix, column); - cf.addAtom(params.makeTombstoneForOverwrite(name.slice())); - } - if (value != UNSET_VALUE) - Adder.doAdd(cf, prefix, column, params, value); + if (value == UNSET_VALUE) + return; + + // delete + add + if (column.type.isMultiCell()) + params.setComplexDeletionTimeForOverwrite(column, writer); + Adder.doAdd(value, clustering, writer, column, params); } } @@ -280,15 +277,15 @@ public abstract class Sets super(column, t); } - public void execute(ByteBuffer rowKey, ColumnFamily cf, Composite prefix, UpdateParameters params) throws InvalidRequestException + public void execute(DecoratedKey partitionKey, Clustering clustering, Row.Writer writer, UpdateParameters params) throws InvalidRequestException { assert column.type.isMultiCell() : "Attempted to add items to a frozen set"; Term.Terminal value = t.bind(params.options); if (value != UNSET_VALUE) - doAdd(cf, prefix, column, params, value); + doAdd(value, clustering, writer, column, params); } - static void doAdd(ColumnFamily cf, Composite prefix, ColumnDefinition column, UpdateParameters params, Term.Terminal value) throws InvalidRequestException + static void doAdd(Term.Terminal value, Clustering clustering, Row.Writer writer, ColumnDefinition column, UpdateParameters params) throws InvalidRequestException { if (column.type.isMultiCell()) { @@ -299,18 +296,17 @@ public abstract class Sets { if (bb == ByteBufferUtil.UNSET_BYTE_BUFFER) continue; - CellName cellName = cf.getComparator().create(prefix, column, bb); - cf.addColumn(params.makeColumn(cellName, ByteBufferUtil.EMPTY_BYTE_BUFFER)); + + params.addCell(clustering, column, writer, CellPath.create(bb), ByteBufferUtil.EMPTY_BYTE_BUFFER); } } else { // for frozen sets, we're overwriting the whole cell - CellName cellName = cf.getComparator().create(prefix, column); if (value == null) - cf.addAtom(params.makeTombstone(cellName)); + params.addTombstone(column, writer); else - cf.addColumn(params.makeColumn(cellName, value.get(Server.CURRENT_VERSION))); + params.addCell(clustering, column, writer, value.get(Server.CURRENT_VERSION)); } } } @@ -323,7 +319,7 @@ public abstract class Sets super(column, t); } - public void execute(ByteBuffer rowKey, ColumnFamily cf, Composite prefix, UpdateParameters params) throws InvalidRequestException + public void execute(DecoratedKey partitionKey, Clustering clustering, Row.Writer writer, UpdateParameters params) throws InvalidRequestException { assert column.type.isMultiCell() : "Attempted to remove items from a frozen set"; @@ -337,7 +333,7 @@ public abstract class Sets : Collections.singleton(value.get(params.options.getProtocolVersion())); for (ByteBuffer bb : toDiscard) - cf.addColumn(params.makeTombstone(cf.getComparator().create(prefix, column, bb))); + params.addTombstone(column, writer, CellPath.create(bb)); } } @@ -348,15 +344,14 @@ public abstract class Sets super(column, k); } - public void execute(ByteBuffer rowKey, ColumnFamily cf, Composite prefix, UpdateParameters params) throws InvalidRequestException + public void execute(DecoratedKey partitionKey, Clustering clustering, Row.Writer writer, UpdateParameters params) throws InvalidRequestException { assert column.type.isMultiCell() : "Attempted to delete a single element in a frozen set"; Term.Terminal elt = t.bind(params.options); if (elt == null) throw new InvalidRequestException("Invalid null set element"); - CellName cellName = cf.getComparator().create(prefix, column, elt.get(params.options.getProtocolVersion())); - cf.addColumn(params.makeTombstone(cellName)); + params.addTombstone(column, writer, CellPath.create(elt.get(params.options.getProtocolVersion()))); } } } diff --git a/src/java/org/apache/cassandra/cql3/SingleColumnRelation.java b/src/java/org/apache/cassandra/cql3/SingleColumnRelation.java index c4c48aaa5f..885a2e2745 100644 --- a/src/java/org/apache/cassandra/cql3/SingleColumnRelation.java +++ b/src/java/org/apache/cassandra/cql3/SingleColumnRelation.java @@ -140,13 +140,13 @@ public final class SingleColumnRelation extends Relation ColumnDefinition columnDef = toColumnDefinition(cfm, entity); if (mapKey == null) { - Term term = toTerm(toReceivers(columnDef), value, cfm.ksName, boundNames); - return new SingleColumnRestriction.EQ(columnDef, term); + Term term = toTerm(toReceivers(columnDef, cfm.isDense()), value, cfm.ksName, boundNames); + return new SingleColumnRestriction.EQRestriction(columnDef, term); } - List receivers = toReceivers(columnDef); + List receivers = toReceivers(columnDef, cfm.isDense()); Term entryKey = toTerm(Collections.singletonList(receivers.get(0)), mapKey, cfm.ksName, boundNames); Term entryValue = toTerm(Collections.singletonList(receivers.get(1)), value, cfm.ksName, boundNames); - return new SingleColumnRestriction.Contains(columnDef, entryKey, entryValue); + return new SingleColumnRestriction.ContainsRestriction(columnDef, entryKey, entryValue); } @Override @@ -154,14 +154,14 @@ public final class SingleColumnRelation extends Relation VariableSpecifications boundNames) throws InvalidRequestException { ColumnDefinition columnDef = cfm.getColumnDefinition(getEntity().prepare(cfm)); - List receivers = toReceivers(columnDef); + List receivers = toReceivers(columnDef, cfm.isDense()); List terms = toTerms(receivers, inValues, cfm.ksName, boundNames); if (terms == null) { Term term = toTerm(receivers, value, cfm.ksName, boundNames); - return new SingleColumnRestriction.InWithMarker(columnDef, (Lists.Marker) term); + return new SingleColumnRestriction.InRestrictionWithMarker(columnDef, (Lists.Marker) term); } - return new SingleColumnRestriction.InWithValues(columnDef, terms); + return new SingleColumnRestriction.InRestrictionWithValues(columnDef, terms); } @Override @@ -171,8 +171,8 @@ public final class SingleColumnRelation extends Relation boolean inclusive) throws InvalidRequestException { ColumnDefinition columnDef = toColumnDefinition(cfm, entity); - Term term = toTerm(toReceivers(columnDef), value, cfm.ksName, boundNames); - return new SingleColumnRestriction.Slice(columnDef, bound, inclusive, term); + Term term = toTerm(toReceivers(columnDef, cfm.isDense()), value, cfm.ksName, boundNames); + return new SingleColumnRestriction.SliceRestriction(columnDef, bound, inclusive, term); } @Override @@ -181,22 +181,23 @@ public final class SingleColumnRelation extends Relation boolean isKey) throws InvalidRequestException { ColumnDefinition columnDef = toColumnDefinition(cfm, entity); - Term term = toTerm(toReceivers(columnDef), value, cfm.ksName, boundNames); - return new SingleColumnRestriction.Contains(columnDef, term, isKey); + Term term = toTerm(toReceivers(columnDef, cfm.isDense()), value, cfm.ksName, boundNames); + return new SingleColumnRestriction.ContainsRestriction(columnDef, term, isKey); } /** * Returns the receivers for this relation. * @param columnDef the column definition + * @param isDense whether the table is a dense one * * @return the receivers for the specified relation. * @throws InvalidRequestException if the relation is invalid */ - private List toReceivers(ColumnDefinition columnDef) throws InvalidRequestException + private List toReceivers(ColumnDefinition columnDef, boolean isDense) throws InvalidRequestException { ColumnSpecification receiver = columnDef; - checkFalse(columnDef.isCompactValue(), + checkFalse(!columnDef.isPrimaryKeyColumn() && isDense, "Predicates on the non-primary-key column (%s) of a COMPACT table are not yet supported", columnDef.name); diff --git a/src/java/org/apache/cassandra/cql3/TokenRelation.java b/src/java/org/apache/cassandra/cql3/TokenRelation.java index 5896fae130..14bd5e0a62 100644 --- a/src/java/org/apache/cassandra/cql3/TokenRelation.java +++ b/src/java/org/apache/cassandra/cql3/TokenRelation.java @@ -69,7 +69,7 @@ public final class TokenRelation extends Relation { List columnDefs = getColumnDefinitions(cfm); Term term = toTerm(toReceivers(cfm, columnDefs), value, cfm.ksName, boundNames); - return new TokenRestriction.EQ(cfm.getKeyValidatorAsCType(), columnDefs, term); + return new TokenRestriction.EQRestriction(cfm.getKeyValidatorAsClusteringComparator(), columnDefs, term); } @Override @@ -86,7 +86,7 @@ public final class TokenRelation extends Relation { List columnDefs = getColumnDefinitions(cfm); Term term = toTerm(toReceivers(cfm, columnDefs), value, cfm.ksName, boundNames); - return new TokenRestriction.Slice(cfm.getKeyValidatorAsCType(), columnDefs, bound, inclusive, term); + return new TokenRestriction.SliceRestriction(cfm.getKeyValidatorAsClusteringComparator(), columnDefs, bound, inclusive, term); } @Override diff --git a/src/java/org/apache/cassandra/cql3/UntypedResultSet.java b/src/java/org/apache/cassandra/cql3/UntypedResultSet.java index 49e0d86c53..f481f5cd20 100644 --- a/src/java/org/apache/cassandra/cql3/UntypedResultSet.java +++ b/src/java/org/apache/cassandra/cql3/UntypedResultSet.java @@ -24,9 +24,16 @@ import java.util.*; import com.google.common.collect.AbstractIterator; +import org.apache.cassandra.config.CFMetaData; +import org.apache.cassandra.config.ColumnDefinition; import org.apache.cassandra.cql3.statements.SelectStatement; +import org.apache.cassandra.db.*; +import org.apache.cassandra.db.rows.*; +import org.apache.cassandra.db.partitions.PartitionIterator; import org.apache.cassandra.db.marshal.*; import org.apache.cassandra.service.pager.QueryPager; +import org.apache.cassandra.transport.Server; +import org.apache.cassandra.utils.FBUtilities; /** a utility for doing internal cql-based queries */ public abstract class UntypedResultSet implements Iterable @@ -174,11 +181,16 @@ public abstract class UntypedResultSet implements Iterable protected Row computeNext() { + int nowInSec = FBUtilities.nowInSeconds(); while (currentPage == null || !currentPage.hasNext()) { if (pager.isExhausted()) return endOfData(); - currentPage = select.process(pager.fetchPage(pageSize)).rows.iterator(); + + try (ReadOrderGroup orderGroup = pager.startOrderGroup(); PartitionIterator iter = pager.fetchPageInternal(pageSize, orderGroup)) + { + currentPage = select.process(iter, nowInSec).rows.iterator(); + } } return new Row(metadata, currentPage.next()); } @@ -208,6 +220,37 @@ public abstract class UntypedResultSet implements Iterable data.put(names.get(i).name.toString(), columns.get(i)); } + public static Row fromInternalRow(CFMetaData metadata, DecoratedKey key, org.apache.cassandra.db.rows.Row row) + { + Map data = new HashMap<>(); + + ByteBuffer[] keyComponents = SelectStatement.getComponents(metadata, key); + for (ColumnDefinition def : metadata.partitionKeyColumns()) + data.put(def.name.toString(), keyComponents[def.position()]); + + Clustering clustering = row.clustering(); + for (ColumnDefinition def : metadata.clusteringColumns()) + data.put(def.name.toString(), clustering.get(def.position())); + + for (ColumnDefinition def : metadata.partitionColumns()) + { + if (def.isComplex()) + { + Iterator cells = row.getCells(def); + if (cells != null) + data.put(def.name.toString(), ((CollectionType)def.type).serializeForNativeProtocol(def, cells, Server.VERSION_3)); + } + else + { + Cell cell = row.getCell(def); + if (cell != null) + data.put(def.name.toString(), cell.value()); + } + } + + return new Row(data); + } + public boolean has(String column) { // Note that containsKey won't work because we may have null values diff --git a/src/java/org/apache/cassandra/cql3/UpdateParameters.java b/src/java/org/apache/cassandra/cql3/UpdateParameters.java index e412585660..66f6b43d6e 100644 --- a/src/java/org/apache/cassandra/cql3/UpdateParameters.java +++ b/src/java/org/apache/cassandra/cql3/UpdateParameters.java @@ -18,15 +18,18 @@ package org.apache.cassandra.cql3; import java.nio.ByteBuffer; -import java.util.Collections; -import java.util.List; import java.util.Map; import org.apache.cassandra.config.CFMetaData; +import org.apache.cassandra.config.ColumnDefinition; import org.apache.cassandra.db.*; -import org.apache.cassandra.db.composites.CellName; -import org.apache.cassandra.db.filter.ColumnSlice; +import org.apache.cassandra.db.filter.ColumnFilter; +import org.apache.cassandra.db.rows.*; +import org.apache.cassandra.db.context.CounterContext; +import org.apache.cassandra.db.index.SecondaryIndexManager; +import org.apache.cassandra.db.partitions.*; import org.apache.cassandra.exceptions.InvalidRequestException; +import org.apache.cassandra.utils.ByteBufferUtil; import org.apache.cassandra.utils.FBUtilities; /** @@ -36,22 +39,39 @@ public class UpdateParameters { public final CFMetaData metadata; public final QueryOptions options; - public final long timestamp; - private final int ttl; - public final int localDeletionTime; + + private final LivenessInfo defaultLiveness; + private final LivenessInfo deletionLiveness; + private final DeletionTime deletionTime; + + private final SecondaryIndexManager indexManager; // For lists operation that require a read-before-write. Will be null otherwise. - private final Map prefetchedLists; + private final Map prefetchedRows; - public UpdateParameters(CFMetaData metadata, QueryOptions options, long timestamp, int ttl, Map prefetchedLists) + public UpdateParameters(CFMetaData metadata, QueryOptions options, long timestamp, int ttl, Map prefetchedRows, boolean validateIndexedColumns) throws InvalidRequestException { this.metadata = metadata; this.options = options; - this.timestamp = timestamp; - this.ttl = ttl; - this.localDeletionTime = (int)(System.currentTimeMillis() / 1000); - this.prefetchedLists = prefetchedLists; + + int nowInSec = FBUtilities.nowInSeconds(); + this.defaultLiveness = SimpleLivenessInfo.forUpdate(timestamp, ttl, nowInSec, metadata); + this.deletionLiveness = SimpleLivenessInfo.forDeletion(timestamp, nowInSec); + this.deletionTime = new SimpleDeletionTime(timestamp, nowInSec); + + this.prefetchedRows = prefetchedRows; + + // Index column validation triggers a call to Keyspace.open() which we want to be able to avoid in some case (e.g. when using CQLSSTableWriter) + if (validateIndexedColumns) + { + SecondaryIndexManager manager = Keyspace.openAndGetStore(metadata).indexManager; + indexManager = manager.hasIndexes() ? manager : null; + } + else + { + indexManager = null; + } // We use MIN_VALUE internally to mean the absence of of timestamp (in Selection, in sstable stats, ...), so exclude // it to avoid potential confusion. @@ -59,48 +79,106 @@ public class UpdateParameters throw new InvalidRequestException(String.format("Out of bound timestamp, must be in [%d, %d]", Long.MIN_VALUE + 1, Long.MAX_VALUE)); } - public Cell makeColumn(CellName name, ByteBuffer value) throws InvalidRequestException + public void newPartition(DecoratedKey partitionKey) throws InvalidRequestException { - QueryProcessor.validateCellName(name, metadata.comparator); - return AbstractCell.create(name, value, timestamp, ttl, metadata); + if (indexManager != null) + indexManager.validate(partitionKey); } - public Cell makeCounter(CellName name, long delta) throws InvalidRequestException - { - QueryProcessor.validateCellName(name, metadata.comparator); - return new BufferCounterUpdateCell(name, delta, FBUtilities.timestampMicros()); - } - - public Cell makeTombstone(CellName name) throws InvalidRequestException + public void writeClustering(Clustering clustering, Row.Writer writer) throws InvalidRequestException { - QueryProcessor.validateCellName(name, metadata.comparator); - return new BufferDeletedCell(name, localDeletionTime, timestamp); + if (indexManager != null) + indexManager.validate(clustering); + + if (metadata.isDense() && !metadata.isCompound()) + { + // If it's a COMPACT STORAGE table with a single clustering column, the clustering value is + // translated in Thrift to the full Thrift column name, and for backward compatibility we + // don't want to allow that to be empty (even though this would be fine for the storage engine). + assert clustering.size() == 1; + ByteBuffer value = clustering.get(0); + if (value == null || !value.hasRemaining()) + throw new InvalidRequestException("Invalid empty or null value for column " + metadata.clusteringColumns().get(0).name); + } + + Rows.writeClustering(clustering, writer); } - public RangeTombstone makeRangeTombstone(ColumnSlice slice) throws InvalidRequestException + public void writePartitionKeyLivenessInfo(Row.Writer writer) { - QueryProcessor.validateComposite(slice.start, metadata.comparator); - QueryProcessor.validateComposite(slice.finish, metadata.comparator); - return new RangeTombstone(slice.start, slice.finish, timestamp, localDeletionTime); + writer.writePartitionKeyLivenessInfo(defaultLiveness); } - public RangeTombstone makeTombstoneForOverwrite(ColumnSlice slice) throws InvalidRequestException + public void writeRowDeletion(Row.Writer writer) { - QueryProcessor.validateComposite(slice.start, metadata.comparator); - QueryProcessor.validateComposite(slice.finish, metadata.comparator); - return new RangeTombstone(slice.start, slice.finish, timestamp - 1, localDeletionTime); + writer.writeRowDeletion(deletionTime); } - public List getPrefetchedList(ByteBuffer rowKey, ColumnIdentifier cql3ColumnName) + public void addTombstone(ColumnDefinition column, Row.Writer writer) throws InvalidRequestException { - if (prefetchedLists == null) - return Collections.emptyList(); + addTombstone(column, writer, null); + } - CQL3Row row = prefetchedLists.get(rowKey); - if (row == null) - return Collections.emptyList(); + public void addTombstone(ColumnDefinition column, Row.Writer writer, CellPath path) throws InvalidRequestException + { + writer.writeCell(column, false, ByteBufferUtil.EMPTY_BYTE_BUFFER, deletionLiveness, path); + } - List cql3List = row.getMultiCellColumn(cql3ColumnName); - return (cql3List == null) ? Collections.emptyList() : cql3List; + public void addCell(Clustering clustering, ColumnDefinition column, Row.Writer writer, ByteBuffer value) throws InvalidRequestException + { + addCell(clustering, column, writer, null, value); + } + + public void addCell(Clustering clustering, ColumnDefinition column, Row.Writer writer, CellPath path, ByteBuffer value) throws InvalidRequestException + { + if (indexManager != null) + indexManager.validate(column, value, path); + + writer.writeCell(column, false, value, defaultLiveness, path); + } + + public void addCounter(ColumnDefinition column, Row.Writer writer, long increment) throws InvalidRequestException + { + assert defaultLiveness.ttl() == LivenessInfo.NO_TTL; + + // In practice, the actual CounterId (and clock really) that we use doesn't matter, because we will + // actually ignore it in CounterMutation when we do the read-before-write to create the actual value + // that is applied. In other words, this is not the actual value that will be written to the memtable + // because this will be replaced in CounterMutation.updateWithCurrentValue(). + // As an aside, since we don't care about the CounterId/clock, we used to only send the incremement, + // but that makes things a bit more complex as this means we need to be able to distinguish inside + // PartitionUpdate between counter updates that has been processed by CounterMutation and those that + // haven't. + ByteBuffer value = CounterContext.instance().createLocal(increment); + writer.writeCell(column, true, value, defaultLiveness, null); + } + + public void setComplexDeletionTime(ColumnDefinition column, Row.Writer writer) + { + writer.writeComplexDeletion(column, deletionTime); + } + + public void setComplexDeletionTimeForOverwrite(ColumnDefinition column, Row.Writer writer) + { + writer.writeComplexDeletion(column, new SimpleDeletionTime(deletionTime.markedForDeleteAt() - 1, deletionTime.localDeletionTime())); + } + + public DeletionTime deletionTime() + { + return deletionTime; + } + + public RangeTombstone makeRangeTombstone(CBuilder cbuilder) + { + return new RangeTombstone(cbuilder.buildSlice(), deletionTime); + } + + public Row getPrefetchedRow(DecoratedKey key, Clustering clustering) + { + if (prefetchedRows == null) + return null; + + Partition partition = prefetchedRows.get(key); + return partition == null ? null : partition.searchIterator(ColumnFilter.selection(partition.columns()), false).next(clustering); } } diff --git a/src/java/org/apache/cassandra/cql3/functions/TokenFct.java b/src/java/org/apache/cassandra/cql3/functions/TokenFct.java index 9d50a972f6..c76b588d7c 100644 --- a/src/java/org/apache/cassandra/cql3/functions/TokenFct.java +++ b/src/java/org/apache/cassandra/cql3/functions/TokenFct.java @@ -22,7 +22,8 @@ import java.util.List; import org.apache.cassandra.config.CFMetaData; import org.apache.cassandra.config.ColumnDefinition; -import org.apache.cassandra.db.composites.CBuilder; +import org.apache.cassandra.cql3.statements.SelectStatement; +import org.apache.cassandra.db.CBuilder; import org.apache.cassandra.db.marshal.AbstractType; import org.apache.cassandra.dht.IPartitioner; import org.apache.cassandra.exceptions.InvalidRequestException; @@ -52,7 +53,7 @@ public class TokenFct extends NativeScalarFunction public ByteBuffer execute(int protocolVersion, List parameters) throws InvalidRequestException { - CBuilder builder = cfm.getKeyValidatorAsCType().builder(); + CBuilder builder = CBuilder.create(cfm.getKeyValidatorAsClusteringComparator()); for (int i = 0; i < parameters.size(); i++) { ByteBuffer bb = parameters.get(i); @@ -60,6 +61,6 @@ public class TokenFct extends NativeScalarFunction return null; builder.add(bb); } - return partitioner.getTokenFactory().toByteArray(partitioner.getToken(builder.build().toByteBuffer())); + return partitioner.getTokenFactory().toByteArray(partitioner.getToken(CFMetaData.serializePartitionKey(builder.build()))); } } diff --git a/src/java/org/apache/cassandra/cql3/restrictions/AbstractPrimaryKeyRestrictions.java b/src/java/org/apache/cassandra/cql3/restrictions/AbstractPrimaryKeyRestrictions.java index 2eaa3868b3..f1b5a502f9 100644 --- a/src/java/org/apache/cassandra/cql3/restrictions/AbstractPrimaryKeyRestrictions.java +++ b/src/java/org/apache/cassandra/cql3/restrictions/AbstractPrimaryKeyRestrictions.java @@ -18,11 +18,12 @@ package org.apache.cassandra.cql3.restrictions; import java.nio.ByteBuffer; -import java.util.List; +import java.util.*; import org.apache.cassandra.cql3.QueryOptions; import org.apache.cassandra.cql3.statements.Bound; -import org.apache.cassandra.db.composites.CType; +import org.apache.cassandra.db.ClusteringPrefix; +import org.apache.cassandra.db.ClusteringComparator; import org.apache.cassandra.exceptions.InvalidRequestException; /** @@ -33,11 +34,11 @@ abstract class AbstractPrimaryKeyRestrictions extends AbstractRestriction implem /** * The composite type. */ - protected final CType ctype; + protected final ClusteringComparator comparator; - public AbstractPrimaryKeyRestrictions(CType ctype) + public AbstractPrimaryKeyRestrictions(ClusteringComparator comparator) { - this.ctype = ctype; + this.comparator = comparator; } @Override diff --git a/src/java/org/apache/cassandra/cql3/restrictions/AbstractRestriction.java b/src/java/org/apache/cassandra/cql3/restrictions/AbstractRestriction.java index 64c94f45d5..0f56fd9a43 100644 --- a/src/java/org/apache/cassandra/cql3/restrictions/AbstractRestriction.java +++ b/src/java/org/apache/cassandra/cql3/restrictions/AbstractRestriction.java @@ -22,7 +22,7 @@ import java.nio.ByteBuffer; import org.apache.cassandra.cql3.ColumnSpecification; import org.apache.cassandra.cql3.QueryOptions; import org.apache.cassandra.cql3.statements.Bound; -import org.apache.cassandra.db.composites.CompositesBuilder; +import org.apache.cassandra.db.MultiCBuilder; import org.apache.cassandra.exceptions.InvalidRequestException; import static org.apache.cassandra.cql3.statements.RequestValidations.checkBindValueSet; @@ -77,7 +77,7 @@ abstract class AbstractRestriction implements Restriction } @Override - public CompositesBuilder appendBoundTo(CompositesBuilder builder, Bound bound, QueryOptions options) + public MultiCBuilder appendBoundTo(MultiCBuilder builder, Bound bound, QueryOptions options) { return appendTo(builder, options); } @@ -87,14 +87,4 @@ abstract class AbstractRestriction implements Restriction { return true; } - - protected static ByteBuffer validateIndexedValue(ColumnSpecification columnSpec, - ByteBuffer value) - throws InvalidRequestException - { - checkNotNull(value, "Unsupported null value for indexed column %s", columnSpec.name); - checkBindValueSet(value, "Unsupported unset value for indexed column %s", columnSpec.name); - checkFalse(value.remaining() > 0xFFFF, "Index expression values may not be larger than 64K"); - return value; - } } diff --git a/src/java/org/apache/cassandra/cql3/restrictions/ForwardingPrimaryKeyRestrictions.java b/src/java/org/apache/cassandra/cql3/restrictions/ForwardingPrimaryKeyRestrictions.java index 03c6cbc23c..91359e6efa 100644 --- a/src/java/org/apache/cassandra/cql3/restrictions/ForwardingPrimaryKeyRestrictions.java +++ b/src/java/org/apache/cassandra/cql3/restrictions/ForwardingPrimaryKeyRestrictions.java @@ -18,16 +18,15 @@ package org.apache.cassandra.cql3.restrictions; import java.nio.ByteBuffer; -import java.util.Collection; -import java.util.List; +import java.util.*; import org.apache.cassandra.config.ColumnDefinition; import org.apache.cassandra.cql3.QueryOptions; import org.apache.cassandra.cql3.functions.Function; import org.apache.cassandra.cql3.statements.Bound; -import org.apache.cassandra.db.IndexExpression; -import org.apache.cassandra.db.composites.Composite; -import org.apache.cassandra.db.composites.CompositesBuilder; +import org.apache.cassandra.db.*; +import org.apache.cassandra.db.Slice; +import org.apache.cassandra.db.filter.RowFilter; import org.apache.cassandra.db.index.SecondaryIndexManager; import org.apache.cassandra.exceptions.InvalidRequestException; @@ -87,15 +86,15 @@ abstract class ForwardingPrimaryKeyRestrictions implements PrimaryKeyRestriction } @Override - public CompositesBuilder appendTo(CompositesBuilder builder, QueryOptions options) + public MultiCBuilder appendTo(MultiCBuilder builder, QueryOptions options) { return getDelegate().appendTo(builder, options); } @Override - public List valuesAsComposites(QueryOptions options) throws InvalidRequestException + public NavigableSet valuesAsClustering(QueryOptions options) throws InvalidRequestException { - return getDelegate().valuesAsComposites(options); + return getDelegate().valuesAsClustering(options); } @Override @@ -105,13 +104,13 @@ abstract class ForwardingPrimaryKeyRestrictions implements PrimaryKeyRestriction } @Override - public List boundsAsComposites(Bound bound, QueryOptions options) throws InvalidRequestException + public SortedSet boundsAsClustering(Bound bound, QueryOptions options) throws InvalidRequestException { - return getDelegate().boundsAsComposites(bound, options); + return getDelegate().boundsAsClustering(bound, options); } @Override - public CompositesBuilder appendBoundTo(CompositesBuilder builder, Bound bound, QueryOptions options) + public MultiCBuilder appendBoundTo(MultiCBuilder builder, Bound bound, QueryOptions options) { return getDelegate().appendBoundTo(builder, bound, options); } @@ -177,10 +176,8 @@ abstract class ForwardingPrimaryKeyRestrictions implements PrimaryKeyRestriction } @Override - public void addIndexExpressionTo(List expressions, - SecondaryIndexManager indexManager, - QueryOptions options) throws InvalidRequestException + public void addRowFilterTo(RowFilter filter, SecondaryIndexManager indexManager, QueryOptions options) throws InvalidRequestException { - getDelegate().addIndexExpressionTo(expressions, indexManager, options); + getDelegate().addRowFilterTo(filter, indexManager, options); } } diff --git a/src/java/org/apache/cassandra/cql3/restrictions/MultiColumnRestriction.java b/src/java/org/apache/cassandra/cql3/restrictions/MultiColumnRestriction.java index c4bce4c8f1..e48a22bca5 100644 --- a/src/java/org/apache/cassandra/cql3/restrictions/MultiColumnRestriction.java +++ b/src/java/org/apache/cassandra/cql3/restrictions/MultiColumnRestriction.java @@ -24,8 +24,8 @@ import org.apache.cassandra.config.ColumnDefinition; import org.apache.cassandra.cql3.*; import org.apache.cassandra.cql3.functions.Function; import org.apache.cassandra.cql3.statements.Bound; -import org.apache.cassandra.db.IndexExpression; -import org.apache.cassandra.db.composites.CompositesBuilder; +import org.apache.cassandra.db.*; +import org.apache.cassandra.db.filter.RowFilter; import org.apache.cassandra.db.index.SecondaryIndex; import org.apache.cassandra.db.index.SecondaryIndexManager; import org.apache.cassandra.exceptions.InvalidRequestException; @@ -108,7 +108,7 @@ public abstract class MultiColumnRestriction extends AbstractRestriction { for (ColumnDefinition columnDef : columnDefs) { - SecondaryIndex index = indexManager.getIndexForColumn(columnDef.name.bytes); + SecondaryIndex index = indexManager.getIndexForColumn(columnDef); if (index != null && isSupportedBy(index)) return true; } @@ -124,11 +124,11 @@ public abstract class MultiColumnRestriction extends AbstractRestriction */ protected abstract boolean isSupportedBy(SecondaryIndex index); - public static class EQ extends MultiColumnRestriction + public static class EQRestriction extends MultiColumnRestriction { protected final Term value; - public EQ(List columnDefs, Term value) + public EQRestriction(List columnDefs, Term value) { super(columnDefs); this.value = value; @@ -160,7 +160,7 @@ public abstract class MultiColumnRestriction extends AbstractRestriction } @Override - public CompositesBuilder appendTo(CompositesBuilder builder, QueryOptions options) + public MultiCBuilder appendTo(MultiCBuilder builder, QueryOptions options) { Tuples.Value t = ((Tuples.Value) value.bind(options)); List values = t.getElements(); @@ -173,9 +173,7 @@ public abstract class MultiColumnRestriction extends AbstractRestriction } @Override - public final void addIndexExpressionTo(List expressions, - SecondaryIndexManager indexManager, - QueryOptions options) throws InvalidRequestException + public final void addRowFilterTo(RowFilter filter, SecondaryIndexManager indexMananger, QueryOptions options) throws InvalidRequestException { Tuples.Value t = ((Tuples.Value) value.bind(options)); List values = t.getElements(); @@ -183,19 +181,23 @@ public abstract class MultiColumnRestriction extends AbstractRestriction for (int i = 0, m = columnDefs.size(); i < m; i++) { ColumnDefinition columnDef = columnDefs.get(i); - ByteBuffer component = validateIndexedValue(columnDef, values.get(i)); - expressions.add(new IndexExpression(columnDef.name.bytes, Operator.EQ, component)); + filter.add(columnDef, Operator.EQ, values.get(i)); } } } - public abstract static class IN extends MultiColumnRestriction + public abstract static class INRestriction extends MultiColumnRestriction { + public INRestriction(List columnDefs) + { + super(columnDefs); + } + /** * {@inheritDoc} */ @Override - public CompositesBuilder appendTo(CompositesBuilder builder, QueryOptions options) + public MultiCBuilder appendTo(MultiCBuilder builder, QueryOptions options) { List> splitInValues = splitValues(options); builder.addAllElementsToAll(splitInValues); @@ -205,11 +207,6 @@ public abstract class MultiColumnRestriction extends AbstractRestriction return builder; } - public IN(List columnDefs) - { - super(columnDefs); - } - @Override public boolean isIN() { @@ -230,9 +227,9 @@ public abstract class MultiColumnRestriction extends AbstractRestriction } @Override - public final void addIndexExpressionTo(List expressions, - SecondaryIndexManager indexManager, - QueryOptions options) throws InvalidRequestException + public final void addRowFilterTo(RowFilter filter, + SecondaryIndexManager indexManager, + QueryOptions options) throws InvalidRequestException { List> splitInValues = splitValues(options); checkTrue(splitInValues.size() == 1, "IN restrictions are not supported on indexed columns"); @@ -241,8 +238,7 @@ public abstract class MultiColumnRestriction extends AbstractRestriction for (int i = 0, m = columnDefs.size(); i < m; i++) { ColumnDefinition columnDef = columnDefs.get(i); - ByteBuffer component = validateIndexedValue(columnDef, values.get(i)); - expressions.add(new IndexExpression(columnDef.name.bytes, Operator.EQ, component)); + filter.add(columnDef, Operator.EQ, values.get(i)); } } @@ -253,11 +249,11 @@ public abstract class MultiColumnRestriction extends AbstractRestriction * An IN restriction that has a set of terms for in values. * For example: "SELECT ... WHERE (a, b, c) IN ((1, 2, 3), (4, 5, 6))" or "WHERE (a, b, c) IN (?, ?)" */ - public static class InWithValues extends MultiColumnRestriction.IN + public static class InRestrictionWithValues extends INRestriction { protected final List values; - public InWithValues(List columnDefs, List values) + public InRestrictionWithValues(List columnDefs, List values) { super(columnDefs); this.values = values; @@ -292,11 +288,11 @@ public abstract class MultiColumnRestriction extends AbstractRestriction * An IN restriction that uses a single marker for a set of IN values that are tuples. * For example: "SELECT ... WHERE (a, b, c) IN ?" */ - public static class InWithMarker extends MultiColumnRestriction.IN + public static class InRestrictionWithMarker extends INRestriction { protected final AbstractMarker marker; - public InWithMarker(List columnDefs, AbstractMarker marker) + public InRestrictionWithMarker(List columnDefs, AbstractMarker marker) { super(columnDefs); this.marker = marker; @@ -324,16 +320,16 @@ public abstract class MultiColumnRestriction extends AbstractRestriction } } - public static class Slice extends MultiColumnRestriction + public static class SliceRestriction extends MultiColumnRestriction { private final TermSlice slice; - public Slice(List columnDefs, Bound bound, boolean inclusive, Term term) + public SliceRestriction(List columnDefs, Bound bound, boolean inclusive, Term term) { this(columnDefs, TermSlice.newInstance(bound, inclusive, term)); } - private Slice(List columnDefs, TermSlice slice) + private SliceRestriction(List columnDefs, TermSlice slice) { super(columnDefs); this.slice = slice; @@ -346,13 +342,13 @@ public abstract class MultiColumnRestriction extends AbstractRestriction } @Override - public CompositesBuilder appendTo(CompositesBuilder builder, QueryOptions options) + public MultiCBuilder appendTo(MultiCBuilder builder, QueryOptions options) { throw new UnsupportedOperationException(); } @Override - public CompositesBuilder appendBoundTo(CompositesBuilder builder, Bound bound, QueryOptions options) + public MultiCBuilder appendBoundTo(MultiCBuilder builder, Bound bound, QueryOptions options) { List vals = componentBounds(bound, options); @@ -395,7 +391,7 @@ public abstract class MultiColumnRestriction extends AbstractRestriction "Column \"%s\" cannot be restricted by both an equality and an inequality relation", getColumnsInCommons(otherRestriction)); - Slice otherSlice = (Slice) otherRestriction; + SliceRestriction otherSlice = (SliceRestriction) otherRestriction; if (!getFirstColumn().equals(otherRestriction.getFirstColumn())) { @@ -414,13 +410,13 @@ public abstract class MultiColumnRestriction extends AbstractRestriction getColumnsInCommons(otherRestriction)); List newColumnDefs = columnDefs.size() >= otherSlice.columnDefs.size() ? columnDefs : otherSlice.columnDefs; - return new Slice(newColumnDefs, slice.merge(otherSlice.slice)); + return new SliceRestriction(newColumnDefs, slice.merge(otherSlice.slice)); } @Override - public final void addIndexExpressionTo(List expressions, - SecondaryIndexManager indexManager, - QueryOptions options) throws InvalidRequestException + public final void addRowFilterTo(RowFilter filter, + SecondaryIndexManager indexManager, + QueryOptions options) throws InvalidRequestException { throw invalidRequest("Slice restrictions are not supported on indexed columns"); } diff --git a/src/java/org/apache/cassandra/cql3/restrictions/PrimaryKeyRestrictionSet.java b/src/java/org/apache/cassandra/cql3/restrictions/PrimaryKeyRestrictionSet.java index b49d774222..39322ff2d9 100644 --- a/src/java/org/apache/cassandra/cql3/restrictions/PrimaryKeyRestrictionSet.java +++ b/src/java/org/apache/cassandra/cql3/restrictions/PrimaryKeyRestrictionSet.java @@ -18,19 +18,18 @@ package org.apache.cassandra.cql3.restrictions; import java.nio.ByteBuffer; -import java.util.Collection; -import java.util.Collections; -import java.util.List; +import java.util.*; +import org.apache.cassandra.config.CFMetaData; import org.apache.cassandra.config.ColumnDefinition; import org.apache.cassandra.cql3.QueryOptions; import org.apache.cassandra.cql3.functions.Function; import org.apache.cassandra.cql3.statements.Bound; -import org.apache.cassandra.db.IndexExpression; -import org.apache.cassandra.db.composites.*; -import org.apache.cassandra.db.composites.Composite.EOC; +import org.apache.cassandra.db.*; +import org.apache.cassandra.db.filter.RowFilter; import org.apache.cassandra.db.index.SecondaryIndexManager; import org.apache.cassandra.exceptions.InvalidRequestException; +import org.apache.cassandra.utils.FBUtilities; import static org.apache.cassandra.cql3.statements.RequestValidations.checkFalse; import static org.apache.cassandra.cql3.statements.RequestValidations.invalidRequest; @@ -65,18 +64,25 @@ final class PrimaryKeyRestrictionSet extends AbstractPrimaryKeyRestrictions */ private boolean contains; - public PrimaryKeyRestrictionSet(CType ctype) + /** + * true if the restrictions corresponding to a partition key, false if it's clustering columns. + */ + private boolean isPartitionKey; + + public PrimaryKeyRestrictionSet(ClusteringComparator comparator, boolean isPartitionKey) { - super(ctype); + super(comparator); this.restrictions = new RestrictionSet(); this.eq = true; + this.isPartitionKey = isPartitionKey; } private PrimaryKeyRestrictionSet(PrimaryKeyRestrictionSet primaryKeyRestrictions, Restriction restriction) throws InvalidRequestException { - super(primaryKeyRestrictions.ctype); + super(primaryKeyRestrictions.comparator); this.restrictions = primaryKeyRestrictions.restrictions.addRestriction(restriction); + this.isPartitionKey = primaryKeyRestrictions.isPartitionKey; if (!primaryKeyRestrictions.isEmpty()) { @@ -104,6 +110,15 @@ final class PrimaryKeyRestrictionSet extends AbstractPrimaryKeyRestrictions this.eq = true; } + private List toByteBuffers(SortedSet clusterings) + { + // It's currently a tad hard to follow that this is only called for partition key so we should fix that + List l = new ArrayList<>(clusterings.size()); + for (ClusteringPrefix clustering : clusterings) + l.add(CFMetaData.serializePartitionKey(clustering)); + return l; + } + @Override public boolean isSlice() { @@ -161,13 +176,13 @@ final class PrimaryKeyRestrictionSet extends AbstractPrimaryKeyRestrictions } @Override - public List valuesAsComposites(QueryOptions options) throws InvalidRequestException + public NavigableSet valuesAsClustering(QueryOptions options) throws InvalidRequestException { - return appendTo(new CompositesBuilder(ctype), options).build(); + return appendTo(MultiCBuilder.create(comparator), options).build(); } @Override - public CompositesBuilder appendTo(CompositesBuilder builder, QueryOptions options) + public MultiCBuilder appendTo(MultiCBuilder builder, QueryOptions options) { for (Restriction r : restrictions) { @@ -179,27 +194,22 @@ final class PrimaryKeyRestrictionSet extends AbstractPrimaryKeyRestrictions } @Override - public CompositesBuilder appendBoundTo(CompositesBuilder builder, Bound bound, QueryOptions options) + public MultiCBuilder appendBoundTo(MultiCBuilder builder, Bound bound, QueryOptions options) { throw new UnsupportedOperationException(); } @Override - public List boundsAsComposites(Bound bound, QueryOptions options) throws InvalidRequestException + public SortedSet boundsAsClustering(Bound bound, QueryOptions options) throws InvalidRequestException { - CompositesBuilder builder = new CompositesBuilder(ctype); - // The end-of-component of composite doesn't depend on whether the - // component type is reversed or not (i.e. the ReversedType is applied - // to the component comparator but not to the end-of-component itself), - // it only depends on whether the slice is reversed + MultiCBuilder builder = MultiCBuilder.create(comparator); int keyPosition = 0; for (Restriction r : restrictions) { ColumnDefinition def = r.getFirstColumn(); - // In a restriction, we always have Bound.START < Bound.END for the "base" comparator. - // So if we're doing a reverse slice, we must inverse the bounds when giving them as start and end of the slice filter. - // But if the actual comparator itself is reversed, we must inversed the bounds too. + // The bound of this method is refering to the clustering order. So if said clustering order + // is reversed for this column, we should reverse the restriction we use. Bound b = !def.isReversedType() ? bound : bound.reverse(); if (keyPosition != def.position() || r.isContains()) break; @@ -211,49 +221,41 @@ final class PrimaryKeyRestrictionSet extends AbstractPrimaryKeyRestrictions // There wasn't any non EQ relation on that key, we select all records having the preceding component as prefix. // For composites, if there was preceding component and we're computing the end, we must change the last component // End-Of-Component, otherwise we would be selecting only one record. - return builder.buildWithEOC(bound.isEnd() ? EOC.END : EOC.START); + return builder.buildBound(bound.isStart(), true); } r.appendBoundTo(builder, b, options); - Composite.EOC eoc = eocFor(r, bound, b); - return builder.buildWithEOC(eoc); + return builder.buildBound(bound.isStart(), r.isInclusive(b)); } r.appendBoundTo(builder, b, options); if (builder.hasMissingElements()) - return Collections.emptyList(); + return FBUtilities.emptySortedSet(comparator); keyPosition = r.getLastColumn().position() + 1; } - // Means no relation at all or everything was an equal - // Note: if the builder is "full", there is no need to use the end-of-component bit. For columns selection, - // it would be harmless to do it. However, we use this method got the partition key too. And when a query - // with 2ndary index is done, and with the the partition provided with an EQ, we'll end up here, and in that - // case using the eoc would be bad, since for the random partitioner we have no guarantee that - // prefix.end() will sort after prefix (see #5240). - EOC eoc = !builder.hasRemaining() ? EOC.NONE : (bound.isEnd() ? EOC.END : EOC.START); - return builder.buildWithEOC(eoc); + + // Everything was an equal (or there was nothing) + return builder.buildBound(bound.isStart(), true); } @Override public List values(QueryOptions options) throws InvalidRequestException { - return Composites.toByteBuffers(valuesAsComposites(options)); + if (!isPartitionKey) + throw new UnsupportedOperationException(); + + return toByteBuffers(valuesAsClustering(options)); } @Override public List bounds(Bound b, QueryOptions options) throws InvalidRequestException { - return Composites.toByteBuffers(boundsAsComposites(b, options)); - } + if (!isPartitionKey) + throw new UnsupportedOperationException(); - private static Composite.EOC eocFor(Restriction r, Bound eocBound, Bound inclusiveBound) - { - if (eocBound.isStart()) - return r.isInclusive(inclusiveBound) ? Composite.EOC.NONE : Composite.EOC.END; - - return r.isInclusive(inclusiveBound) ? Composite.EOC.END : Composite.EOC.START; + return toByteBuffers(boundsAsClustering(b, options)); } @Override @@ -279,30 +281,24 @@ final class PrimaryKeyRestrictionSet extends AbstractPrimaryKeyRestrictions } @Override - public void addIndexExpressionTo(List expressions, - SecondaryIndexManager indexManager, - QueryOptions options) throws InvalidRequestException + public void addRowFilterTo(RowFilter filter, + SecondaryIndexManager indexManager, + QueryOptions options) throws InvalidRequestException { - Boolean clusteringColumns = null; int position = 0; for (Restriction restriction : restrictions) { ColumnDefinition columnDef = restriction.getFirstColumn(); - // PrimaryKeyRestrictionSet contains only one kind of column, either partition key or clustering columns. - // Therefore we only need to check the column kind once. All the other columns will be of the same kind. - if (clusteringColumns == null) - clusteringColumns = columnDef.isClusteringColumn() ? Boolean.TRUE : Boolean.FALSE; - // We ignore all the clustering columns that can be handled by slices. - if (clusteringColumns && !restriction.isContains()&& position == columnDef.position()) + if (!isPartitionKey && !restriction.isContains()&& position == columnDef.position()) { position = restriction.getLastColumn().position() + 1; if (!restriction.hasSupportingIndex(indexManager)) continue; } - restriction.addIndexExpressionTo(expressions, indexManager, options); + restriction.addRowFilterTo(filter, indexManager, options); } } diff --git a/src/java/org/apache/cassandra/cql3/restrictions/PrimaryKeyRestrictions.java b/src/java/org/apache/cassandra/cql3/restrictions/PrimaryKeyRestrictions.java index 7d7b492073..6a14182f45 100644 --- a/src/java/org/apache/cassandra/cql3/restrictions/PrimaryKeyRestrictions.java +++ b/src/java/org/apache/cassandra/cql3/restrictions/PrimaryKeyRestrictions.java @@ -19,10 +19,13 @@ package org.apache.cassandra.cql3.restrictions; import java.nio.ByteBuffer; import java.util.List; +import java.util.NavigableSet; +import java.util.SortedSet; import org.apache.cassandra.cql3.QueryOptions; import org.apache.cassandra.cql3.statements.Bound; -import org.apache.cassandra.db.composites.Composite; +import org.apache.cassandra.db.Clustering; +import org.apache.cassandra.db.Slice; import org.apache.cassandra.exceptions.InvalidRequestException; /** @@ -36,9 +39,9 @@ interface PrimaryKeyRestrictions extends Restriction, Restrictions public List values(QueryOptions options) throws InvalidRequestException; - public List valuesAsComposites(QueryOptions options) throws InvalidRequestException; + public NavigableSet valuesAsClustering(QueryOptions options) throws InvalidRequestException; public List bounds(Bound b, QueryOptions options) throws InvalidRequestException; - public List boundsAsComposites(Bound bound, QueryOptions options) throws InvalidRequestException; + public SortedSet boundsAsClustering(Bound bound, QueryOptions options) throws InvalidRequestException; } diff --git a/src/java/org/apache/cassandra/cql3/restrictions/Restriction.java b/src/java/org/apache/cassandra/cql3/restrictions/Restriction.java index c115a3b962..370a0f20b1 100644 --- a/src/java/org/apache/cassandra/cql3/restrictions/Restriction.java +++ b/src/java/org/apache/cassandra/cql3/restrictions/Restriction.java @@ -18,14 +18,13 @@ package org.apache.cassandra.cql3.restrictions; import java.util.Collection; -import java.util.List; import org.apache.cassandra.config.ColumnDefinition; import org.apache.cassandra.cql3.QueryOptions; import org.apache.cassandra.cql3.functions.Function; import org.apache.cassandra.cql3.statements.Bound; -import org.apache.cassandra.db.IndexExpression; -import org.apache.cassandra.db.composites.CompositesBuilder; +import org.apache.cassandra.db.MultiCBuilder; +import org.apache.cassandra.db.filter.RowFilter; import org.apache.cassandra.db.index.SecondaryIndexManager; import org.apache.cassandra.exceptions.InvalidRequestException; @@ -105,35 +104,34 @@ public interface Restriction public boolean hasSupportingIndex(SecondaryIndexManager indexManager); /** - * Adds to the specified list the IndexExpressions corresponding to this Restriction. + * Adds to the specified row filter the expressions corresponding to this Restriction. * - * @param expressions the list to add the IndexExpressions to + * @param filter the row filter to add expressions to * @param indexManager the secondary index manager * @param options the query options - * @throws InvalidRequestException if this Restriction cannot be converted into - * IndexExpressions + * @throws InvalidRequestException if this Restriction cannot be converted into a row filter */ - public void addIndexExpressionTo(List expressions, - SecondaryIndexManager indexManager, - QueryOptions options) - throws InvalidRequestException; + public void addRowFilterTo(RowFilter filter, + SecondaryIndexManager indexManager, + QueryOptions options) + throws InvalidRequestException; /** * Appends the values of this Restriction to the specified builder. * - * @param builder the CompositesBuilder to append to. + * @param builder the MultiCBuilder to append to. * @param options the query options - * @return the CompositesBuilder + * @return the MultiCBuilder */ - public CompositesBuilder appendTo(CompositesBuilder builder, QueryOptions options); + public MultiCBuilder appendTo(MultiCBuilder builder, QueryOptions options); /** * Appends the values of the Restriction for the specified bound to the specified builder. * - * @param builder the CompositesBuilder to append to. + * @param builder the MultiCBuilder to append to. * @param bound the bound * @param options the query options - * @return the CompositesBuilder + * @return the MultiCBuilder */ - public CompositesBuilder appendBoundTo(CompositesBuilder builder, Bound bound, QueryOptions options); + public MultiCBuilder appendBoundTo(MultiCBuilder builder, Bound bound, QueryOptions options); } diff --git a/src/java/org/apache/cassandra/cql3/restrictions/RestrictionSet.java b/src/java/org/apache/cassandra/cql3/restrictions/RestrictionSet.java index 6bf7666219..10fae1386b 100644 --- a/src/java/org/apache/cassandra/cql3/restrictions/RestrictionSet.java +++ b/src/java/org/apache/cassandra/cql3/restrictions/RestrictionSet.java @@ -24,8 +24,8 @@ import com.google.common.collect.Iterables; import org.apache.cassandra.config.ColumnDefinition; import org.apache.cassandra.cql3.QueryOptions; import org.apache.cassandra.cql3.functions.Function; -import org.apache.cassandra.cql3.restrictions.SingleColumnRestriction.Contains; -import org.apache.cassandra.db.IndexExpression; +import org.apache.cassandra.cql3.restrictions.SingleColumnRestriction.ContainsRestriction; +import org.apache.cassandra.db.filter.RowFilter; import org.apache.cassandra.db.index.SecondaryIndexManager; import org.apache.cassandra.exceptions.InvalidRequestException; @@ -66,12 +66,10 @@ final class RestrictionSet implements Restrictions, Iterable } @Override - public final void addIndexExpressionTo(List expressions, - SecondaryIndexManager indexManager, - QueryOptions options) throws InvalidRequestException + public final void addRowFilterTo(RowFilter filter, SecondaryIndexManager indexManager, QueryOptions options) throws InvalidRequestException { for (Restriction restriction : restrictions.values()) - restriction.addIndexExpressionTo(expressions, indexManager, options); + restriction.addRowFilterTo(filter, indexManager, options); } @Override @@ -245,7 +243,7 @@ final class RestrictionSet implements Restrictions, Iterable { if (restriction.isContains()) { - Contains contains = (Contains) restriction; + ContainsRestriction contains = (ContainsRestriction) restriction; numberOfContains += (contains.numberOfValues() + contains.numberOfKeys() + contains.numberOfEntries()); } } diff --git a/src/java/org/apache/cassandra/cql3/restrictions/Restrictions.java b/src/java/org/apache/cassandra/cql3/restrictions/Restrictions.java index ab81bf7b3a..88d0bb6c6a 100644 --- a/src/java/org/apache/cassandra/cql3/restrictions/Restrictions.java +++ b/src/java/org/apache/cassandra/cql3/restrictions/Restrictions.java @@ -18,12 +18,11 @@ package org.apache.cassandra.cql3.restrictions; import java.util.Collection; -import java.util.List; import org.apache.cassandra.config.ColumnDefinition; import org.apache.cassandra.cql3.QueryOptions; import org.apache.cassandra.cql3.functions.Function; -import org.apache.cassandra.db.IndexExpression; +import org.apache.cassandra.db.filter.RowFilter; import org.apache.cassandra.db.index.SecondaryIndexManager; import org.apache.cassandra.exceptions.InvalidRequestException; @@ -54,18 +53,14 @@ interface Restrictions public boolean hasSupportingIndex(SecondaryIndexManager indexManager); /** - * Adds to the specified list the IndexExpressions corresponding to this Restriction. + * Adds to the specified row filter the expressions corresponding to this Restrictions. * - * @param expressions the list to add the IndexExpressions to + * @param filter the row filter to add expressions to * @param indexManager the secondary index manager * @param options the query options - * @throws InvalidRequestException if this Restriction cannot be converted into - * IndexExpressions + * @throws InvalidRequestException if this Restrictions cannot be converted into a row filter */ - public void addIndexExpressionTo(List expressions, - SecondaryIndexManager indexManager, - QueryOptions options) - throws InvalidRequestException; + public void addRowFilterTo(RowFilter filter, SecondaryIndexManager indexManager, QueryOptions options) throws InvalidRequestException; /** * Checks if this PrimaryKeyRestrictionSet is empty or not. diff --git a/src/java/org/apache/cassandra/cql3/restrictions/ReversedPrimaryKeyRestrictions.java b/src/java/org/apache/cassandra/cql3/restrictions/ReversedPrimaryKeyRestrictions.java deleted file mode 100644 index 9b3316117d..0000000000 --- a/src/java/org/apache/cassandra/cql3/restrictions/ReversedPrimaryKeyRestrictions.java +++ /dev/null @@ -1,77 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.apache.cassandra.cql3.restrictions; - -import java.nio.ByteBuffer; -import java.util.Collections; -import java.util.List; - -import org.apache.cassandra.cql3.QueryOptions; -import org.apache.cassandra.cql3.statements.Bound; -import org.apache.cassandra.db.composites.Composite; -import org.apache.cassandra.exceptions.InvalidRequestException; - -/** - * PrimaryKeyRestrictions decorator that reverse the slices. - */ -final class ReversedPrimaryKeyRestrictions extends ForwardingPrimaryKeyRestrictions -{ - /** - * The decorated restrictions. - */ - private PrimaryKeyRestrictions restrictions; - - public ReversedPrimaryKeyRestrictions(PrimaryKeyRestrictions restrictions) - { - this.restrictions = restrictions; - } - - @Override - public PrimaryKeyRestrictions mergeWith(Restriction restriction) throws InvalidRequestException - { - return new ReversedPrimaryKeyRestrictions(this.restrictions.mergeWith(restriction)); - } - - @Override - public List bounds(Bound bound, QueryOptions options) throws InvalidRequestException - { - List buffers = restrictions.bounds(bound.reverse(), options); - Collections.reverse(buffers); - return buffers; - } - - @Override - public List boundsAsComposites(Bound bound, QueryOptions options) throws InvalidRequestException - { - List composites = restrictions.boundsAsComposites(bound.reverse(), options); - Collections.reverse(composites); - return composites; - } - - @Override - public boolean isInclusive(Bound bound) - { - return this.restrictions.isInclusive(bound.reverse()); - } - - @Override - protected PrimaryKeyRestrictions getDelegate() - { - return this.restrictions; - } -} diff --git a/src/java/org/apache/cassandra/cql3/restrictions/SingleColumnRestriction.java b/src/java/org/apache/cassandra/cql3/restrictions/SingleColumnRestriction.java index d32f585343..f96b7a054e 100644 --- a/src/java/org/apache/cassandra/cql3/restrictions/SingleColumnRestriction.java +++ b/src/java/org/apache/cassandra/cql3/restrictions/SingleColumnRestriction.java @@ -27,11 +27,10 @@ import org.apache.cassandra.cql3.*; import org.apache.cassandra.cql3.Term.Terminal; import org.apache.cassandra.cql3.functions.Function; import org.apache.cassandra.cql3.statements.Bound; -import org.apache.cassandra.db.IndexExpression; -import org.apache.cassandra.db.composites.CompositesBuilder; +import org.apache.cassandra.db.MultiCBuilder; +import org.apache.cassandra.db.filter.RowFilter; import org.apache.cassandra.db.index.SecondaryIndex; import org.apache.cassandra.db.index.SecondaryIndexManager; -import org.apache.cassandra.db.marshal.CompositeType; import org.apache.cassandra.exceptions.InvalidRequestException; import static org.apache.cassandra.cql3.statements.RequestValidations.checkBindValueSet; @@ -73,7 +72,7 @@ public abstract class SingleColumnRestriction extends AbstractRestriction @Override public boolean hasSupportingIndex(SecondaryIndexManager indexManager) { - SecondaryIndex index = indexManager.getIndexForColumn(columnDef.name.bytes); + SecondaryIndex index = indexManager.getIndexForColumn(columnDef); return index != null && isSupportedBy(index); } @@ -97,11 +96,11 @@ public abstract class SingleColumnRestriction extends AbstractRestriction */ protected abstract boolean isSupportedBy(SecondaryIndex index); - public static final class EQ extends SingleColumnRestriction + public static final class EQRestriction extends SingleColumnRestriction { private final Term value; - public EQ(ColumnDefinition columnDef, Term value) + public EQRestriction(ColumnDefinition columnDef, Term value) { super(columnDef); this.value = value; @@ -120,16 +119,15 @@ public abstract class SingleColumnRestriction extends AbstractRestriction } @Override - public void addIndexExpressionTo(List expressions, - SecondaryIndexManager indexManager, - QueryOptions options) throws InvalidRequestException + public void addRowFilterTo(RowFilter filter, + SecondaryIndexManager indexManager, + QueryOptions options) throws InvalidRequestException { - ByteBuffer buffer = validateIndexedValue(columnDef, value.bindAndGet(options)); - expressions.add(new IndexExpression(columnDef.name.bytes, Operator.EQ, buffer)); + filter.add(columnDef, Operator.EQ, value.bindAndGet(options)); } @Override - public CompositesBuilder appendTo(CompositesBuilder builder, QueryOptions options) + public MultiCBuilder appendTo(MultiCBuilder builder, QueryOptions options) { builder.addElementToAll(value.bindAndGet(options)); checkFalse(builder.containsNull(), "Invalid null value in condition for column %s", columnDef.name); @@ -156,9 +154,9 @@ public abstract class SingleColumnRestriction extends AbstractRestriction } } - public static abstract class IN extends SingleColumnRestriction + public static abstract class INRestriction extends SingleColumnRestriction { - public IN(ColumnDefinition columnDef) + public INRestriction(ColumnDefinition columnDef) { super(columnDef); } @@ -176,7 +174,7 @@ public abstract class SingleColumnRestriction extends AbstractRestriction } @Override - public CompositesBuilder appendTo(CompositesBuilder builder, QueryOptions options) + public MultiCBuilder appendTo(MultiCBuilder builder, QueryOptions options) { builder.addEachElementToAll(getValues(options)); checkFalse(builder.containsNull(), "Invalid null value in condition for column %s", columnDef.name); @@ -185,15 +183,14 @@ public abstract class SingleColumnRestriction extends AbstractRestriction } @Override - public void addIndexExpressionTo(List expressions, - SecondaryIndexManager indexManager, - QueryOptions options) throws InvalidRequestException + public void addRowFilterTo(RowFilter filter, + SecondaryIndexManager indexManager, + QueryOptions options) throws InvalidRequestException { List values = getValues(options); checkTrue(values.size() == 1, "IN restrictions are not supported on indexed columns"); - ByteBuffer value = validateIndexedValue(columnDef, values.get(0)); - expressions.add(new IndexExpression(columnDef.name.bytes, Operator.EQ, value)); + filter.add(columnDef, Operator.EQ, values.get(0)); } @Override @@ -205,11 +202,11 @@ public abstract class SingleColumnRestriction extends AbstractRestriction protected abstract List getValues(QueryOptions options) throws InvalidRequestException; } - public static class InWithValues extends IN + public static class InRestrictionWithValues extends INRestriction { protected final List values; - public InWithValues(ColumnDefinition columnDef, List values) + public InRestrictionWithValues(ColumnDefinition columnDef, List values) { super(columnDef); this.values = values; @@ -237,11 +234,11 @@ public abstract class SingleColumnRestriction extends AbstractRestriction } } - public static class InWithMarker extends IN + public static class InRestrictionWithMarker extends INRestriction { protected final AbstractMarker marker; - public InWithMarker(ColumnDefinition columnDef, AbstractMarker marker) + public InRestrictionWithMarker(ColumnDefinition columnDef, AbstractMarker marker) { super(columnDef); this.marker = marker; @@ -270,11 +267,11 @@ public abstract class SingleColumnRestriction extends AbstractRestriction } } - public static class Slice extends SingleColumnRestriction + public static class SliceRestriction extends SingleColumnRestriction { private final TermSlice slice; - public Slice(ColumnDefinition columnDef, Bound bound, boolean inclusive, Term term) + public SliceRestriction(ColumnDefinition columnDef, Bound bound, boolean inclusive, Term term) { super(columnDef); slice = TermSlice.newInstance(bound, inclusive, term); @@ -293,7 +290,7 @@ public abstract class SingleColumnRestriction extends AbstractRestriction } @Override - public CompositesBuilder appendTo(CompositesBuilder builder, QueryOptions options) + public MultiCBuilder appendTo(MultiCBuilder builder, QueryOptions options) { throw new UnsupportedOperationException(); } @@ -305,7 +302,7 @@ public abstract class SingleColumnRestriction extends AbstractRestriction } @Override - public CompositesBuilder appendBoundTo(CompositesBuilder builder, Bound bound, QueryOptions options) + public MultiCBuilder appendBoundTo(MultiCBuilder builder, Bound bound, QueryOptions options) { ByteBuffer value = slice.bound(bound).bindAndGet(options); checkBindValueSet(value, "Invalid unset value for column %s", columnDef.name); @@ -326,7 +323,7 @@ public abstract class SingleColumnRestriction extends AbstractRestriction "Column \"%s\" cannot be restricted by both an equality and an inequality relation", columnDef.name); - SingleColumnRestriction.Slice otherSlice = (SingleColumnRestriction.Slice) otherRestriction; + SingleColumnRestriction.SliceRestriction otherSlice = (SingleColumnRestriction.SliceRestriction) otherRestriction; checkFalse(hasBound(Bound.START) && otherSlice.hasBound(Bound.START), "More than one restriction was found for the start bound on %s", columnDef.name); @@ -334,27 +331,15 @@ public abstract class SingleColumnRestriction extends AbstractRestriction checkFalse(hasBound(Bound.END) && otherSlice.hasBound(Bound.END), "More than one restriction was found for the end bound on %s", columnDef.name); - return new Slice(columnDef, slice.merge(otherSlice.slice)); + return new SliceRestriction(columnDef, slice.merge(otherSlice.slice)); } @Override - public void addIndexExpressionTo(List expressions, - SecondaryIndexManager indexManager, - QueryOptions options) throws InvalidRequestException + public void addRowFilterTo(RowFilter filter, SecondaryIndexManager indexManager, QueryOptions options) throws InvalidRequestException { for (Bound b : Bound.values()) - { if (hasBound(b)) - { - ByteBuffer value = validateIndexedValue(columnDef, slice.bound(b).bindAndGet(options)); - Operator op = slice.getIndexOperator(b); - // If the underlying comparator for name is reversed, we need to reverse the IndexOperator: user operation - // always refer to the "forward" sorting even if the clustering order is reversed, but the 2ndary code does - // use the underlying comparator as is. - op = columnDef.isReversedType() ? op.reverse() : op; - expressions.add(new IndexExpression(columnDef.name.bytes, op, value)); - } - } + filter.add(columnDef, slice.getIndexOperator(b), slice.bound(b).bindAndGet(options)); } @Override @@ -369,7 +354,7 @@ public abstract class SingleColumnRestriction extends AbstractRestriction return String.format("SLICE%s", slice); } - private Slice(ColumnDefinition columnDef, TermSlice slice) + private SliceRestriction(ColumnDefinition columnDef, TermSlice slice) { super(columnDef); this.slice = slice; @@ -377,14 +362,14 @@ public abstract class SingleColumnRestriction extends AbstractRestriction } // This holds CONTAINS, CONTAINS_KEY, and map[key] = value restrictions because we might want to have any combination of them. - public static final class Contains extends SingleColumnRestriction + public static final class ContainsRestriction extends SingleColumnRestriction { private List values = new ArrayList<>(); // for CONTAINS private List keys = new ArrayList<>(); // for CONTAINS_KEY private List entryKeys = new ArrayList<>(); // for map[key] = value private List entryValues = new ArrayList<>(); // for map[key] = value - public Contains(ColumnDefinition columnDef, Term t, boolean isKey) + public ContainsRestriction(ColumnDefinition columnDef, Term t, boolean isKey) { super(columnDef); if (isKey) @@ -393,7 +378,7 @@ public abstract class SingleColumnRestriction extends AbstractRestriction values.add(t); } - public Contains(ColumnDefinition columnDef, Term mapKey, Term mapValue) + public ContainsRestriction(ColumnDefinition columnDef, Term mapKey, Term mapValue) { super(columnDef); entryKeys.add(mapKey); @@ -401,7 +386,7 @@ public abstract class SingleColumnRestriction extends AbstractRestriction } @Override - public CompositesBuilder appendTo(CompositesBuilder builder, QueryOptions options) + public MultiCBuilder appendTo(MultiCBuilder builder, QueryOptions options) { throw new UnsupportedOperationException(); } @@ -419,33 +404,27 @@ public abstract class SingleColumnRestriction extends AbstractRestriction "Collection column %s can only be restricted by CONTAINS, CONTAINS KEY, or map-entry equality", columnDef.name); - SingleColumnRestriction.Contains newContains = new Contains(columnDef); + SingleColumnRestriction.ContainsRestriction newContains = new ContainsRestriction(columnDef); copyKeysAndValues(this, newContains); - copyKeysAndValues((Contains) otherRestriction, newContains); + copyKeysAndValues((ContainsRestriction) otherRestriction, newContains); return newContains; } @Override - public void addIndexExpressionTo(List expressions, - SecondaryIndexManager indexManager, - QueryOptions options) - throws InvalidRequestException + public void addRowFilterTo(RowFilter filter, SecondaryIndexManager indexManager, QueryOptions options) throws InvalidRequestException { - addExpressionsFor(expressions, bindAndGet(values, options), Operator.CONTAINS); - addExpressionsFor(expressions, bindAndGet(keys, options), Operator.CONTAINS_KEY); - addExpressionsFor(expressions, entries(options), Operator.EQ); - } + for (ByteBuffer value : bindAndGet(values, options)) + filter.add(columnDef, Operator.CONTAINS, value); + for (ByteBuffer key : bindAndGet(keys, options)) + filter.add(columnDef, Operator.CONTAINS_KEY, key); - private void addExpressionsFor(List target, List values, - Operator op) throws InvalidRequestException - { - for (ByteBuffer value : values) - { - validateIndexedValue(columnDef, value); - target.add(new IndexExpression(columnDef.name.bytes, op, value)); - } + List eks = bindAndGet(entryKeys, options); + List evs = bindAndGet(entryValues, options); + assert eks.size() == evs.size(); + for (int i = 0; i < eks.size(); i++) + filter.addMapEquality(columnDef, eks.get(i), Operator.EQ, evs.get(i)); } @Override @@ -502,7 +481,7 @@ public abstract class SingleColumnRestriction extends AbstractRestriction } @Override - public CompositesBuilder appendBoundTo(CompositesBuilder builder, Bound bound, QueryOptions options) + public MultiCBuilder appendBoundTo(MultiCBuilder builder, Bound bound, QueryOptions options) { throw new UnsupportedOperationException(); } @@ -513,20 +492,6 @@ public abstract class SingleColumnRestriction extends AbstractRestriction throw new UnsupportedOperationException(); } - private List entries(QueryOptions options) throws InvalidRequestException - { - List entryBuffers = new ArrayList<>(entryKeys.size()); - List keyBuffers = bindAndGet(entryKeys, options); - List valueBuffers = bindAndGet(entryValues, options); - for (int i = 0; i < entryKeys.size(); i++) - { - if (valueBuffers.get(i) == null) - throw new InvalidRequestException("Unsupported null value for map-entry equality"); - entryBuffers.add(CompositeType.build(keyBuffers.get(i), valueBuffers.get(i))); - } - return entryBuffers; - } - /** * Binds the query options to the specified terms and returns the resulting values. * @@ -549,7 +514,7 @@ public abstract class SingleColumnRestriction extends AbstractRestriction * @param from the Contains to copy from * @param to the Contains to copy to */ - private static void copyKeysAndValues(Contains from, Contains to) + private static void copyKeysAndValues(ContainsRestriction from, ContainsRestriction to) { to.values.addAll(from.values); to.keys.addAll(from.keys); @@ -557,7 +522,7 @@ public abstract class SingleColumnRestriction extends AbstractRestriction to.entryValues.addAll(from.entryValues); } - private Contains(ColumnDefinition columnDef) + private ContainsRestriction(ColumnDefinition columnDef) { super(columnDef); } diff --git a/src/java/org/apache/cassandra/cql3/restrictions/StatementRestrictions.java b/src/java/org/apache/cassandra/cql3/restrictions/StatementRestrictions.java index c10a56a94f..7660c3ed48 100644 --- a/src/java/org/apache/cassandra/cql3/restrictions/StatementRestrictions.java +++ b/src/java/org/apache/cassandra/cql3/restrictions/StatementRestrictions.java @@ -29,7 +29,7 @@ import org.apache.cassandra.cql3.*; import org.apache.cassandra.cql3.functions.Function; import org.apache.cassandra.cql3.statements.Bound; import org.apache.cassandra.db.*; -import org.apache.cassandra.db.composites.Composite; +import org.apache.cassandra.db.filter.RowFilter; import org.apache.cassandra.db.index.SecondaryIndexManager; import org.apache.cassandra.dht.*; import org.apache.cassandra.exceptions.InvalidRequestException; @@ -67,7 +67,7 @@ public final class StatementRestrictions private RestrictionSet nonPrimaryKeyRestrictions; /** - * The restrictions used to build the index expressions + * The restrictions used to build the row filter */ private final List indexRestrictions = new ArrayList<>(); @@ -95,28 +95,28 @@ public final class StatementRestrictions private StatementRestrictions(CFMetaData cfm) { this.cfm = cfm; - this.partitionKeyRestrictions = new PrimaryKeyRestrictionSet(cfm.getKeyValidatorAsCType()); - this.clusteringColumnsRestrictions = new PrimaryKeyRestrictionSet(cfm.comparator); + this.partitionKeyRestrictions = new PrimaryKeyRestrictionSet(cfm.getKeyValidatorAsClusteringComparator(), true); + this.clusteringColumnsRestrictions = new PrimaryKeyRestrictionSet(cfm.comparator, false); this.nonPrimaryKeyRestrictions = new RestrictionSet(); } public StatementRestrictions(CFMetaData cfm, - List whereClause, - VariableSpecifications boundNames, - boolean selectsOnlyStaticColumns, - boolean selectACollection) throws InvalidRequestException + List whereClause, + VariableSpecifications boundNames, + boolean selectsOnlyStaticColumns, + boolean selectACollection, + boolean useFiltering) throws InvalidRequestException { - this.cfm = cfm; - this.partitionKeyRestrictions = new PrimaryKeyRestrictionSet(cfm.getKeyValidatorAsCType()); - this.clusteringColumnsRestrictions = new PrimaryKeyRestrictionSet(cfm.comparator); - this.nonPrimaryKeyRestrictions = new RestrictionSet(); + this(cfm); /* - * WHERE clause. For a given entity, rules are: - EQ relation conflicts with anything else (including a 2nd EQ) - * - Can't have more than one LT(E) relation (resp. GT(E) relation) - IN relation are restricted to row keys - * (for now) and conflicts with anything else (we could allow two IN for the same entity but that doesn't seem - * very useful) - The value_alias cannot be restricted in any way (we don't support wide rows with indexed value - * in CQL so far) + * WHERE clause. For a given entity, rules are: + * - EQ relation conflicts with anything else (including a 2nd EQ) + * - Can't have more than one LT(E) relation (resp. GT(E) relation) + * - IN relation are restricted to row keys (for now) and conflicts with anything else (we could + * allow two IN for the same entity but that doesn't seem very useful) + * - The value_alias cannot be restricted in any way (we don't support wide rows with indexed value + * in CQL so far) */ for (Relation relation : whereClause) addRestriction(relation.toRestriction(cfm, boundNames)); @@ -133,7 +133,7 @@ public final class StatementRestrictions processPartitionKeyRestrictions(hasQueriableIndex); // Some but not all of the partition key columns have been specified; - // hence we need turn these restrictions into index expressions. + // hence we need turn these restrictions into a row filter. if (usesSecondaryIndexing) indexRestrictions.add(partitionKeyRestrictions); @@ -155,7 +155,11 @@ public final class StatementRestrictions // there is restrictions not covered by the PK. if (!nonPrimaryKeyRestrictions.isEmpty()) { - usesSecondaryIndexing = true; + if (hasQueriableIndex) + usesSecondaryIndexing = true; + else if (!useFiltering) + throw new InvalidRequestException("No supported secondary index found for the non primary key columns restrictions"); + indexRestrictions.add(nonPrimaryKeyRestrictions); } @@ -191,6 +195,19 @@ public final class StatementRestrictions nonPrimaryKeyRestrictions = nonPrimaryKeyRestrictions.addRestriction(restriction); } + /** + * Returns the non-PK column that are restricted. + */ + public Set nonPKRestrictedColumns() + { + Set columns = new HashSet<>(); + for (Restrictions r : indexRestrictions) + for (ColumnDefinition def : r.getColumnDefs()) + if (!def.isPrimaryKeyColumn()) + columns.add(def); + return columns; + } + /** * Checks if the restrictions on the partition key is an IN restriction. * @@ -309,17 +326,16 @@ public final class StatementRestrictions usesSecondaryIndexing = true; } - public List getIndexExpressions(SecondaryIndexManager indexManager, - QueryOptions options) throws InvalidRequestException + public RowFilter getRowFilter(SecondaryIndexManager indexManager, QueryOptions options) throws InvalidRequestException { - if (!usesSecondaryIndexing || indexRestrictions.isEmpty()) - return Collections.emptyList(); + if (indexRestrictions.isEmpty()) + return RowFilter.NONE; - List expressions = new ArrayList<>(); + RowFilter filter = RowFilter.create(); for (Restrictions restrictions : indexRestrictions) - restrictions.addIndexExpressionTo(expressions, indexManager, options); + restrictions.addRowFilterTo(filter, indexManager, options); - return expressions; + return filter; } /** @@ -345,8 +361,7 @@ public final class StatementRestrictions private ByteBuffer getPartitionKeyBound(Bound b, QueryOptions options) throws InvalidRequestException { // Deal with unrestricted partition key components (special-casing is required to deal with 2i queries on the - // first - // component of a composite partition key). + // first component of a composite partition key). if (hasPartitionKeyUnrestrictedComponents()) return ByteBufferUtil.EMPTY_BYTE_BUFFER; @@ -361,7 +376,7 @@ public final class StatementRestrictions * @return the partition key bounds * @throws InvalidRequestException if the query is invalid */ - public AbstractBounds getPartitionKeyBounds(QueryOptions options) throws InvalidRequestException + public AbstractBounds getPartitionKeyBounds(QueryOptions options) throws InvalidRequestException { IPartitioner p = StorageService.getPartitioner(); @@ -373,14 +388,14 @@ public final class StatementRestrictions return getPartitionKeyBounds(p, options); } - private AbstractBounds getPartitionKeyBounds(IPartitioner p, + private AbstractBounds getPartitionKeyBounds(IPartitioner p, QueryOptions options) throws InvalidRequestException { ByteBuffer startKeyBytes = getPartitionKeyBound(Bound.START, options); ByteBuffer finishKeyBytes = getPartitionKeyBound(Bound.END, options); - RowPosition startKey = RowPosition.ForKey.get(startKeyBytes, p); - RowPosition finishKey = RowPosition.ForKey.get(finishKeyBytes, p); + PartitionPosition startKey = PartitionPosition.ForKey.get(startKeyBytes, p); + PartitionPosition finishKey = PartitionPosition.ForKey.get(finishKeyBytes, p); if (startKey.compareTo(finishKey) > 0 && !finishKey.isMinimum()) return null; @@ -397,7 +412,7 @@ public final class StatementRestrictions : new ExcludingBounds<>(startKey, finishKey); } - private AbstractBounds getPartitionKeyBoundsForTokenRestrictions(IPartitioner p, + private AbstractBounds getPartitionKeyBoundsForTokenRestrictions(IPartitioner p, QueryOptions options) throws InvalidRequestException { @@ -422,8 +437,8 @@ public final class StatementRestrictions && (cmp > 0 || (cmp == 0 && (!includeStart || !includeEnd)))) return null; - RowPosition start = includeStart ? startToken.minKeyBound() : startToken.maxKeyBound(); - RowPosition end = includeEnd ? endToken.maxKeyBound() : endToken.minKeyBound(); + PartitionPosition start = includeStart ? startToken.minKeyBound() : startToken.maxKeyBound(); + PartitionPosition end = includeEnd ? endToken.maxKeyBound() : endToken.minKeyBound(); return new Range<>(start, end); } @@ -438,17 +453,6 @@ public final class StatementRestrictions return p.getTokenFactory().fromByteArray(value); } - /** - * Checks if the query does not contains any restriction on the clustering columns. - * - * @return true if the query does not contains any restriction on the clustering columns, - * false otherwise. - */ - public boolean hasNoClusteringColumnsRestriction() - { - return clusteringColumnsRestrictions.isEmpty(); - } - /** * Checks if the query has some restrictions on the clustering columns. * @@ -460,39 +464,16 @@ public final class StatementRestrictions return !clusteringColumnsRestrictions.isEmpty(); } - // For non-composite slices, we don't support internally the difference between exclusive and - // inclusive bounds, so we deal with it manually. - public boolean isNonCompositeSliceWithExclusiveBounds() - { - return !cfm.comparator.isCompound() - && clusteringColumnsRestrictions.isSlice() - && (!clusteringColumnsRestrictions.isInclusive(Bound.START) || !clusteringColumnsRestrictions.isInclusive(Bound.END)); - } - /** - * Returns the requested clustering columns as Composites. + * Returns the requested clustering columns. * * @param options the query options - * @return the requested clustering columns as Composites + * @return the requested clustering columns * @throws InvalidRequestException if the query is not valid */ - public List getClusteringColumnsAsComposites(QueryOptions options) throws InvalidRequestException + public NavigableSet getClusteringColumns(QueryOptions options) throws InvalidRequestException { - return clusteringColumnsRestrictions.valuesAsComposites(options); - } - - /** - * Returns the bounds (start or end) of the clustering columns as Composites. - * - * @param b the bound type - * @param options the query options - * @return the bounds (start or end) of the clustering columns as Composites - * @throws InvalidRequestException if the request is not valid - */ - public List getClusteringColumnsBoundsAsComposites(Bound b, - QueryOptions options) throws InvalidRequestException - { - return clusteringColumnsRestrictions.boundsAsComposites(b, options); + return clusteringColumnsRestrictions.valuesAsClustering(options); } /** @@ -503,9 +484,9 @@ public final class StatementRestrictions * @return the bounds (start or end) of the clustering columns * @throws InvalidRequestException if the request is not valid */ - public List getClusteringColumnsBounds(Bound b, QueryOptions options) throws InvalidRequestException + public SortedSet getClusteringColumnsBounds(Bound b, QueryOptions options) throws InvalidRequestException { - return clusteringColumnsRestrictions.bounds(b, options); + return clusteringColumnsRestrictions.boundsAsClustering(b, options); } /** @@ -527,15 +508,9 @@ public final class StatementRestrictions */ public boolean isColumnRange() { - // Due to CASSANDRA-5762, we always do a slice for CQL3 tables (not dense, composite). - // Static CF (non dense but non composite) never entails a column slice however - if (!cfm.comparator.isDense()) - return cfm.comparator.isCompound(); - - // Otherwise (i.e. for compact table where we don't have a row marker anyway and thus don't care about - // CASSANDRA-5762), // it is a range query if it has at least one the column alias for which no relation is defined or is not EQ. - return clusteringColumnsRestrictions.size() < cfm.clusteringColumns().size() || clusteringColumnsRestrictions.isSlice(); + return clusteringColumnsRestrictions.size() < cfm.clusteringColumns().size() + || (!clusteringColumnsRestrictions.isEQ() && !clusteringColumnsRestrictions.isIN()); } /** @@ -564,9 +539,4 @@ public final class StatementRestrictions // so far, 2i means that you've restricted a non static column, so the query is somewhat non-sensical. checkFalse(selectsOnlyStaticColumns, "Queries using 2ndary indexes don't support selecting only static columns"); } - - public void reverse() - { - clusteringColumnsRestrictions = new ReversedPrimaryKeyRestrictions(clusteringColumnsRestrictions); - } } diff --git a/src/java/org/apache/cassandra/cql3/restrictions/TokenFilter.java b/src/java/org/apache/cassandra/cql3/restrictions/TokenFilter.java index bd04610afe..2ab54ce8cc 100644 --- a/src/java/org/apache/cassandra/cql3/restrictions/TokenFilter.java +++ b/src/java/org/apache/cassandra/cql3/restrictions/TokenFilter.java @@ -18,8 +18,7 @@ package org.apache.cassandra.cql3.restrictions; import java.nio.ByteBuffer; -import java.util.ArrayList; -import java.util.List; +import java.util.*; import com.google.common.collect.BoundType; import com.google.common.collect.ImmutableRangeSet; @@ -28,7 +27,7 @@ import com.google.common.collect.RangeSet; import org.apache.cassandra.cql3.QueryOptions; import org.apache.cassandra.cql3.statements.Bound; -import org.apache.cassandra.db.composites.Composite; +import org.apache.cassandra.db.*; import org.apache.cassandra.dht.IPartitioner; import org.apache.cassandra.dht.Token; import org.apache.cassandra.exceptions.InvalidRequestException; @@ -84,7 +83,7 @@ final class TokenFilter extends ForwardingPrimaryKeyRestrictions } @Override - public List valuesAsComposites(QueryOptions options) throws InvalidRequestException + public NavigableSet valuesAsClustering(QueryOptions options) throws InvalidRequestException { throw new UnsupportedOperationException(); } @@ -117,9 +116,9 @@ final class TokenFilter extends ForwardingPrimaryKeyRestrictions } @Override - public List boundsAsComposites(Bound bound, QueryOptions options) throws InvalidRequestException + public SortedSet boundsAsClustering(Bound bound, QueryOptions options) throws InvalidRequestException { - return tokenRestriction.boundsAsComposites(bound, options); + return tokenRestriction.boundsAsClustering(bound, options); } /** diff --git a/src/java/org/apache/cassandra/cql3/restrictions/TokenRestriction.java b/src/java/org/apache/cassandra/cql3/restrictions/TokenRestriction.java index f8cd0dc7ee..bd0ef3ae19 100644 --- a/src/java/org/apache/cassandra/cql3/restrictions/TokenRestriction.java +++ b/src/java/org/apache/cassandra/cql3/restrictions/TokenRestriction.java @@ -18,9 +18,7 @@ package org.apache.cassandra.cql3.restrictions; import java.nio.ByteBuffer; -import java.util.Collection; -import java.util.Collections; -import java.util.List; +import java.util.*; import com.google.common.base.Joiner; @@ -29,10 +27,8 @@ import org.apache.cassandra.cql3.QueryOptions; import org.apache.cassandra.cql3.Term; import org.apache.cassandra.cql3.functions.Function; import org.apache.cassandra.cql3.statements.Bound; -import org.apache.cassandra.db.IndexExpression; -import org.apache.cassandra.db.composites.CType; -import org.apache.cassandra.db.composites.Composite; -import org.apache.cassandra.db.composites.CompositesBuilder; +import org.apache.cassandra.db.*; +import org.apache.cassandra.db.filter.RowFilter; import org.apache.cassandra.db.index.SecondaryIndexManager; import org.apache.cassandra.exceptions.InvalidRequestException; @@ -51,12 +47,12 @@ public abstract class TokenRestriction extends AbstractPrimaryKeyRestrictions /** * Creates a new TokenRestriction that apply to the specified columns. * - * @param ctype the composite type + * @param comparator the clustering comparator * @param columnDefs the definition of the columns to which apply the token restriction */ - public TokenRestriction(CType ctype, List columnDefs) + public TokenRestriction(ClusteringComparator comparator, List columnDefs) { - super(ctype); + super(comparator); this.columnDefs = columnDefs; } @@ -91,27 +87,25 @@ public abstract class TokenRestriction extends AbstractPrimaryKeyRestrictions } @Override - public final void addIndexExpressionTo(List expressions, - SecondaryIndexManager indexManager, - QueryOptions options) + public void addRowFilterTo(RowFilter filter, SecondaryIndexManager indexManager, QueryOptions options) { throw new UnsupportedOperationException("Index expression cannot be created for token restriction"); } @Override - public CompositesBuilder appendTo(CompositesBuilder builder, QueryOptions options) + public MultiCBuilder appendTo(MultiCBuilder builder, QueryOptions options) { throw new UnsupportedOperationException(); } @Override - public List valuesAsComposites(QueryOptions options) throws InvalidRequestException + public NavigableSet valuesAsClustering(QueryOptions options) throws InvalidRequestException { throw new UnsupportedOperationException(); } @Override - public List boundsAsComposites(Bound bound, QueryOptions options) throws InvalidRequestException + public SortedSet boundsAsClustering(Bound bound, QueryOptions options) throws InvalidRequestException { throw new UnsupportedOperationException(); } @@ -153,16 +147,16 @@ public abstract class TokenRestriction extends AbstractPrimaryKeyRestrictions if (restriction instanceof PrimaryKeyRestrictions) return (PrimaryKeyRestrictions) restriction; - return new PrimaryKeyRestrictionSet(ctype).mergeWith(restriction); + return new PrimaryKeyRestrictionSet(comparator, true).mergeWith(restriction); } - public static final class EQ extends TokenRestriction + public static final class EQRestriction extends TokenRestriction { private final Term value; - public EQ(CType ctype, List columnDefs, Term value) + public EQRestriction(ClusteringComparator comparator, List columnDefs, Term value) { - super(ctype, columnDefs); + super(comparator, columnDefs); this.value = value; } @@ -192,13 +186,13 @@ public abstract class TokenRestriction extends AbstractPrimaryKeyRestrictions } } - public static class Slice extends TokenRestriction + public static class SliceRestriction extends TokenRestriction { private final TermSlice slice; - public Slice(CType ctype, List columnDefs, Bound bound, boolean inclusive, Term term) + public SliceRestriction(ClusteringComparator comparator, List columnDefs, Bound bound, boolean inclusive, Term term) { - super(ctype, columnDefs); + super(comparator, columnDefs); slice = TermSlice.newInstance(bound, inclusive, term); } @@ -246,7 +240,7 @@ public abstract class TokenRestriction extends AbstractPrimaryKeyRestrictions throw invalidRequest("Columns \"%s\" cannot be restricted by both an equality and an inequality relation", getColumnNamesAsString()); - TokenRestriction.Slice otherSlice = (TokenRestriction.Slice) otherRestriction; + TokenRestriction.SliceRestriction otherSlice = (TokenRestriction.SliceRestriction) otherRestriction; if (hasBound(Bound.START) && otherSlice.hasBound(Bound.START)) throw invalidRequest("More than one restriction was found for the start bound on %s", @@ -256,7 +250,7 @@ public abstract class TokenRestriction extends AbstractPrimaryKeyRestrictions throw invalidRequest("More than one restriction was found for the end bound on %s", getColumnNamesAsString()); - return new Slice(ctype, columnDefs, slice.merge(otherSlice.slice)); + return new SliceRestriction(comparator, columnDefs, slice.merge(otherSlice.slice)); } @Override @@ -264,10 +258,9 @@ public abstract class TokenRestriction extends AbstractPrimaryKeyRestrictions { return String.format("SLICE%s", slice); } - - private Slice(CType ctype, List columnDefs, TermSlice slice) + private SliceRestriction(ClusteringComparator comparator, List columnDefs, TermSlice slice) { - super(ctype, columnDefs); + super(comparator, columnDefs); this.slice = slice; } } diff --git a/src/java/org/apache/cassandra/cql3/selection/Selection.java b/src/java/org/apache/cassandra/cql3/selection/Selection.java index 25278dfb6a..52ff6575ef 100644 --- a/src/java/org/apache/cassandra/cql3/selection/Selection.java +++ b/src/java/org/apache/cassandra/cql3/selection/Selection.java @@ -29,9 +29,7 @@ import org.apache.cassandra.config.CFMetaData; import org.apache.cassandra.config.ColumnDefinition; import org.apache.cassandra.cql3.*; import org.apache.cassandra.cql3.functions.Function; -import org.apache.cassandra.db.Cell; -import org.apache.cassandra.db.CounterCell; -import org.apache.cassandra.db.ExpiringCell; +import org.apache.cassandra.db.rows.Cell; import org.apache.cassandra.db.context.CounterContext; import org.apache.cassandra.db.marshal.UTF8Type; import org.apache.cassandra.exceptions.InvalidRequestException; @@ -120,9 +118,6 @@ public abstract class Selection */ public boolean containsACollection() { - if (!cfm.comparator.hasCollections()) - return false; - for (ColumnDefinition def : getColumns()) if (def.type.isCollection() && def.type.isMultiCell()) return true; @@ -237,9 +232,9 @@ public abstract class Selection return columnMapping; } - public ResultSetBuilder resultSetBuilder(long now, boolean isJson) throws InvalidRequestException + public ResultSetBuilder resultSetBuilder(boolean isJons) throws InvalidRequestException { - return new ResultSetBuilder(now, isJson); + return new ResultSetBuilder(isJons); } public abstract boolean isAggregate(); @@ -277,18 +272,22 @@ public abstract class Selection List current; final long[] timestamps; final int[] ttls; - final long now; private final boolean isJson; - private ResultSetBuilder(long now, boolean isJson) throws InvalidRequestException + private ResultSetBuilder(boolean isJson) throws InvalidRequestException { this.resultSet = new ResultSet(getResultMetadata(isJson).copy(), new ArrayList>()); this.selectors = newSelectors(); this.timestamps = collectTimestamps ? new long[columns.size()] : null; this.ttls = collectTTLs ? new int[columns.size()] : null; - this.now = now; this.isJson = isJson; + + // We use MIN_VALUE to indicate no timestamp and -1 for no ttl + if (timestamps != null) + Arrays.fill(timestamps, Long.MIN_VALUE); + if (ttls != null) + Arrays.fill(ttls, -1); } public void add(ByteBuffer v) @@ -296,25 +295,28 @@ public abstract class Selection current.add(v); } - public void add(Cell c) + public void add(Cell c, int nowInSec) { - current.add(isDead(c) ? null : value(c)); + if (c == null) + { + current.add(null); + return; + } + + current.add(value(c)); + if (timestamps != null) - { - timestamps[current.size() - 1] = isDead(c) ? Long.MIN_VALUE : c.timestamp(); - } + timestamps[current.size() - 1] = c.livenessInfo().timestamp(); + if (ttls != null) - { - int ttl = -1; - if (!isDead(c) && c instanceof ExpiringCell) - ttl = c.getLocalDeletionTime() - (int) (now / 1000); - ttls[current.size() - 1] = ttl; - } + ttls[current.size() - 1] = c.livenessInfo().remainingTTL(nowInSec); } - private boolean isDead(Cell c) + private ByteBuffer value(Cell c) { - return c == null || !c.isLive(now); + return c.isCounterCell() + ? ByteBufferUtil.bytes(CounterContext.instance().total(c.value())) + : c.value(); } public void newRow(int protocolVersion) throws InvalidRequestException @@ -378,13 +380,6 @@ public abstract class Selection sb.append("}"); return Collections.singletonList(UTF8Type.instance.getSerializer().serialize(sb.toString())); } - - private ByteBuffer value(Cell c) - { - return (c instanceof CounterCell) - ? ByteBufferUtil.bytes(CounterContext.instance().total(c.value())) - : c.value(); - } } private static interface Selectors diff --git a/src/java/org/apache/cassandra/cql3/selection/Selector.java b/src/java/org/apache/cassandra/cql3/selection/Selector.java index 9b7f0ba938..4e4a2207f6 100644 --- a/src/java/org/apache/cassandra/cql3/selection/Selector.java +++ b/src/java/org/apache/cassandra/cql3/selection/Selector.java @@ -59,7 +59,7 @@ public abstract class Selector implements AssignmentTestable { return new ColumnSpecification(cfm.ksName, cfm.cfName, - new ColumnIdentifier(getColumnName(), true), + ColumnIdentifier.getInterned(getColumnName(), true), getReturnType()); } diff --git a/src/java/org/apache/cassandra/cql3/statements/AlterTableStatement.java b/src/java/org/apache/cassandra/cql3/statements/AlterTableStatement.java index db4c8dcdeb..b6937afd3b 100644 --- a/src/java/org/apache/cassandra/cql3/statements/AlterTableStatement.java +++ b/src/java/org/apache/cassandra/cql3/statements/AlterTableStatement.java @@ -92,12 +92,12 @@ public class AlterTableStatement extends SchemaAlteringStatement { case ADD: assert columnName != null; - if (cfm.comparator.isDense()) + if (cfm.isDense()) throw new InvalidRequestException("Cannot add new column to a COMPACT STORAGE table"); if (isStatic) { - if (!cfm.comparator.isCompound()) + if (!cfm.isCompound()) throw new InvalidRequestException("Static columns are not allowed in COMPACT STORAGE tables"); if (cfm.clusteringColumns().isEmpty()) throw new InvalidRequestException("Static columns are only useful (and thus allowed) if the table has at least one clustering column"); @@ -122,26 +122,23 @@ public class AlterTableStatement extends SchemaAlteringStatement AbstractType type = validator.getType(); if (type.isCollection() && type.isMultiCell()) { - if (!cfm.comparator.supportCollections()) - throw new InvalidRequestException("Cannot use non-frozen collections with a non-composite PRIMARY KEY"); + if (!cfm.isCompound()) + throw new InvalidRequestException("Cannot use non-frozen collections in COMPACT STORAGE tables"); if (cfm.isSuper()) throw new InvalidRequestException("Cannot use non-frozen collections with super column families"); - // If there used to be a collection column with the same name (that has been dropped), it will - // still be appear in the ColumnToCollectionType because or reasons explained on #6276. The same - // reason mean that we can't allow adding a new collection with that name (see the ticket for details). - if (cfm.comparator.hasCollections()) - { - CollectionType previous = cfm.comparator.collectionType() == null ? null : cfm.comparator.collectionType().defined.get(columnName.bytes); - if (previous != null && !type.isCompatibleWith(previous)) - throw new InvalidRequestException(String.format("Cannot add a collection with the name %s " + - "because a collection with the same name and a different type has already been used in the past", columnName)); - } - - cfm.comparator = cfm.comparator.addOrUpdateCollection(columnName, (CollectionType)type); + // If there used to be a collection column with the same name (that has been dropped), we could still have + // some data using the old type, and so we can't allow adding a collection with the same name unless + // the types are compatible (see #6276). + CFMetaData.DroppedColumn dropped = cfm.getDroppedColumns().get(columnName); + // We could have type == null for old dropped columns, in which case we play it safe and refuse + if (dropped != null && (dropped.type == null || (dropped.type instanceof CollectionType && !type.isCompatibleWith(dropped.type)))) + throw new InvalidRequestException(String.format("Cannot add a collection with the name %s " + + "because a collection with the same name and a different type%s has already been used in the past", + columnName, dropped.type == null ? "" : " (" + dropped.type.asCQL3Type() + ")")); } - Integer componentIndex = cfm.comparator.isCompound() ? cfm.comparator.clusteringPrefixSize() : null; + Integer componentIndex = cfm.isCompound() ? cfm.comparator.size() : null; cfm.addColumnDefinition(isStatic ? ColumnDefinition.staticDef(cfm, columnName.bytes, type, componentIndex) : ColumnDefinition.regularDef(cfm, columnName.bytes, type, componentIndex)); @@ -158,28 +155,13 @@ public class AlterTableStatement extends SchemaAlteringStatement case PARTITION_KEY: if (validatorType instanceof CounterColumnType) throw new InvalidRequestException(String.format("counter type is not supported for PRIMARY KEY part %s", columnName)); - if (cfm.getKeyValidator() instanceof CompositeType) - { - List> oldTypes = ((CompositeType) cfm.getKeyValidator()).types; - if (!validatorType.isValueCompatibleWith(oldTypes.get(def.position()))) - throw new ConfigurationException(String.format("Cannot change %s from type %s to type %s: types are incompatible.", - columnName, - oldTypes.get(def.position()).asCQL3Type(), - validator)); - List> newTypes = new ArrayList>(oldTypes); - newTypes.set(def.position(), validatorType); - cfm.keyValidator(CompositeType.getInstance(newTypes)); - } - else - { - if (!validatorType.isValueCompatibleWith(cfm.getKeyValidator())) - throw new ConfigurationException(String.format("Cannot change %s from type %s to type %s: types are incompatible.", - columnName, - cfm.getKeyValidator().asCQL3Type(), - validator)); - cfm.keyValidator(validatorType); - } + AbstractType currentType = cfm.getKeyValidatorAsClusteringComparator().subtype(def.position()); + if (!validatorType.isValueCompatibleWith(currentType)) + throw new ConfigurationException(String.format("Cannot change %s from type %s to type %s: types are incompatible.", + columnName, + currentType.asCQL3Type(), + validator)); break; case CLUSTERING_COLUMN: AbstractType oldType = cfm.comparator.subtype(def.position()); @@ -192,16 +174,6 @@ public class AlterTableStatement extends SchemaAlteringStatement oldType.asCQL3Type(), validator)); - cfm.comparator = cfm.comparator.setSubtype(def.position(), validatorType); - break; - case COMPACT_VALUE: - // See below - if (!validatorType.isValueCompatibleWith(cfm.getDefaultValidator())) - throw new ConfigurationException(String.format("Cannot change %s from type %s to type %s: types are incompatible.", - columnName, - cfm.getDefaultValidator().asCQL3Type(), - validator)); - cfm.defaultValidator(validatorType); break; case REGULAR: case STATIC: @@ -215,14 +187,6 @@ public class AlterTableStatement extends SchemaAlteringStatement columnName, def.type.asCQL3Type(), validator)); - - // For collections, if we alter the type, we need to update the comparator too since it includes - // the type too (note that isValueCompatibleWith above has validated that the new type doesn't - // change the underlying sorting order, but we still don't want to have a discrepancy between the type - // in the comparator and the one in the ColumnDefinition as that would be dodgy). - if (validatorType.isCollection() && validatorType.isMultiCell()) - cfm.comparator = cfm.comparator.addOrUpdateCollection(def.name, (CollectionType)validatorType); - break; } // In any case, we update the column definition @@ -231,7 +195,7 @@ public class AlterTableStatement extends SchemaAlteringStatement case DROP: assert columnName != null; - if (!cfm.isCQL3Table()) + if (!cfm.isCQLTable()) throw new InvalidRequestException("Cannot drop columns from a non-CQL3 table"); if (def == null) throw new InvalidRequestException(String.format("Column %s was not found in table %s", columnName, columnFamily())); @@ -244,7 +208,7 @@ public class AlterTableStatement extends SchemaAlteringStatement case REGULAR: case STATIC: ColumnDefinition toDelete = null; - for (ColumnDefinition columnDef : cfm.regularAndStaticColumns()) + for (ColumnDefinition columnDef : cfm.partitionColumns()) { if (columnDef.name.equals(columnName)) { diff --git a/src/java/org/apache/cassandra/cql3/statements/AlterTypeStatement.java b/src/java/org/apache/cassandra/cql3/statements/AlterTypeStatement.java index 74fafd67a8..4e78bfc6d5 100644 --- a/src/java/org/apache/cassandra/cql3/statements/AlterTypeStatement.java +++ b/src/java/org/apache/cassandra/cql3/statements/AlterTypeStatement.java @@ -23,7 +23,7 @@ import java.util.*; import org.apache.cassandra.auth.Permission; import org.apache.cassandra.config.*; import org.apache.cassandra.cql3.*; -import org.apache.cassandra.db.composites.CellNames; +import org.apache.cassandra.db.ClusteringComparator; import org.apache.cassandra.db.marshal.*; import org.apache.cassandra.exceptions.*; import org.apache.cassandra.service.ClientState; @@ -150,28 +150,6 @@ public abstract class AlterTypeStatement extends SchemaAlteringStatement // We need to update this validator ... cfm.addOrReplaceColumnDefinition(def.withNewType(t)); - - // ... but if it's part of the comparator or key validator, we need to go update those too. - switch (def.kind) - { - case PARTITION_KEY: - cfm.keyValidator(updateWith(cfm.getKeyValidator(), keyspace, toReplace, updated)); - break; - case CLUSTERING_COLUMN: - cfm.comparator = CellNames.fromAbstractType(updateWith(cfm.comparator.asAbstractType(), keyspace, toReplace, updated), cfm.comparator.isDense()); - break; - default: - // If it's a collection, we still want to modify the comparator because the collection is aliased in it - if (def.type instanceof CollectionType && def.type.isMultiCell()) - { - t = updateWith(cfm.comparator.asAbstractType(), keyspace, toReplace, updated); - // If t == null, all relevant comparators were updated via updateWith, which reaches into types and - // collections - if (t != null) - cfm.comparator = CellNames.fromAbstractType(t, cfm.comparator.isDense()); - } - break; - } return true; } @@ -203,23 +181,6 @@ public abstract class AlterTypeStatement extends SchemaAlteringStatement List> updatedTypes = updateTypes(ct.types, keyspace, toReplace, updated); return updatedTypes == null ? null : CompositeType.getInstance(updatedTypes); } - else if (type instanceof ColumnToCollectionType) - { - ColumnToCollectionType ctct = (ColumnToCollectionType)type; - Map updatedTypes = null; - for (Map.Entry entry : ctct.defined.entrySet()) - { - AbstractType t = updateWith(entry.getValue(), keyspace, toReplace, updated); - if (t == null) - continue; - - if (updatedTypes == null) - updatedTypes = new HashMap<>(ctct.defined); - - updatedTypes.put(entry.getKey(), (CollectionType)t); - } - return updatedTypes == null ? null : ColumnToCollectionType.getInstance(updatedTypes); - } else if (type instanceof CollectionType) { if (type instanceof ListType) diff --git a/src/java/org/apache/cassandra/cql3/statements/BatchStatement.java b/src/java/org/apache/cassandra/cql3/statements/BatchStatement.java index 0661b563f0..08a47c0c28 100644 --- a/src/java/org/apache/cassandra/cql3/statements/BatchStatement.java +++ b/src/java/org/apache/cassandra/cql3/statements/BatchStatement.java @@ -32,7 +32,8 @@ import org.apache.cassandra.config.ColumnDefinition; import org.apache.cassandra.config.DatabaseDescriptor; import org.apache.cassandra.cql3.*; import org.apache.cassandra.db.*; -import org.apache.cassandra.db.composites.Composite; +import org.apache.cassandra.db.rows.RowIterator; +import org.apache.cassandra.db.partitions.*; import org.apache.cassandra.exceptions.*; import org.apache.cassandra.service.ClientState; import org.apache.cassandra.service.ClientWarn; @@ -40,6 +41,7 @@ import org.apache.cassandra.service.QueryState; import org.apache.cassandra.service.StorageProxy; import org.apache.cassandra.thrift.Column; import org.apache.cassandra.tracing.Tracing; +import org.apache.cassandra.service.StorageService; import org.apache.cassandra.transport.messages.ResultMessage; import org.apache.cassandra.utils.NoSpamLogger; import org.apache.cassandra.utils.Pair; @@ -57,6 +59,10 @@ public class BatchStatement implements CQLStatement private final int boundTerms; public final Type type; private final List statements; + private final PartitionColumns updatedColumns; + private final PartitionColumns conditionColumns; + private final boolean updatesRegularRows; + private final boolean updatesStaticRow; private final Attributes attrs; private final boolean hasConditions; private static final Logger logger = LoggerFactory.getLogger(BatchStatement.class); @@ -72,14 +78,33 @@ public class BatchStatement implements CQLStatement */ public BatchStatement(int boundTerms, Type type, List statements, Attributes attrs) { - boolean hasConditions = false; - for (ModificationStatement statement : statements) - hasConditions |= statement.hasConditions(); - this.boundTerms = boundTerms; this.type = type; this.statements = statements; this.attrs = attrs; + + boolean hasConditions = false; + PartitionColumns.Builder regularBuilder = PartitionColumns.builder(); + PartitionColumns.Builder conditionBuilder = PartitionColumns.builder(); + boolean updateRegular = false; + boolean updateStatic = false; + + for (ModificationStatement stmt : statements) + { + regularBuilder.addAll(stmt.updatedColumns()); + updateRegular |= stmt.updatesRegularRows(); + if (stmt.hasConditions()) + { + hasConditions = true; + conditionBuilder.addAll(stmt.conditionColumns()); + updateStatic |= stmt.updatesStaticRow(); + } + } + + this.updatedColumns = regularBuilder.build(); + this.conditionColumns = conditionBuilder.build(); + this.updatesRegularRows = updateRegular; + this.updatesStaticRow = updateStatic; this.hasConditions = hasConditions; } @@ -199,6 +224,18 @@ public class BatchStatement implements CQLStatement return ms; } + private PartitionColumns updatedColumns() + { + return updatedColumns; + } + + private int updatedRows() + { + // Note: it's possible for 2 statements to actually apply to the same row, but that's just an estimation + // for sizing our PartitionUpdate backing array, so it's good enough. + return statements.size(); + } + private void addStatementMutations(ModificationStatement statement, QueryOptions options, boolean local, @@ -218,25 +255,33 @@ public class BatchStatement implements CQLStatement // we don't want to recreate mutations every time as this is particularly inefficient when applying // multiple batch to the same partition (see #6737). List keys = statement.buildPartitionKeyNames(options); - Composite clusteringPrefix = statement.createClusteringPrefix(options); - UpdateParameters params = statement.makeUpdateParameters(keys, clusteringPrefix, options, local, now); + CBuilder clustering = statement.createClustering(options); + UpdateParameters params = statement.makeUpdateParameters(keys, clustering, options, local, now); for (ByteBuffer key : keys) { - IMutation mutation = ksMap.get(key); + DecoratedKey dk = StorageService.getPartitioner().decorateKey(key); + IMutation mutation = ksMap.get(dk.getKey()); Mutation mut; if (mutation == null) { - mut = new Mutation(ksName, key); + mut = new Mutation(ksName, dk); mutation = statement.cfm.isCounter() ? new CounterMutation(mut, options.getConsistency()) : mut; - ksMap.put(key, mutation); + ksMap.put(dk.getKey(), mutation); } else { mut = statement.cfm.isCounter() ? ((CounterMutation) mutation).getMutation() : (Mutation) mutation; } - statement.addUpdateForKey(mut.addOrGet(statement.cfm), key, clusteringPrefix, params); + PartitionUpdate upd = mut.get(statement.cfm); + if (upd == null) + { + upd = new PartitionUpdate(statement.cfm, dk, updatedColumns(), updatedRows()); + mut.add(upd); + } + + statement.addUpdateForKey(upd, clustering, params); } } @@ -245,56 +290,55 @@ public class BatchStatement implements CQLStatement * * @param cfs ColumnFamilies that will store the batch's mutations. */ - public static void verifyBatchSize(Iterable cfs) throws InvalidRequestException + public static void verifyBatchSize(Iterable updates) throws InvalidRequestException { long size = 0; long warnThreshold = DatabaseDescriptor.getBatchSizeWarnThreshold(); long failThreshold = DatabaseDescriptor.getBatchSizeFailThreshold(); - for (ColumnFamily cf : cfs) - size += cf.dataSize(); + for (PartitionUpdate update : updates) + size += update.dataSize(); if (size > warnThreshold) { - Set ksCfPairs = new HashSet<>(); - for (ColumnFamily cf : cfs) - ksCfPairs.add(String.format("%s.%s", cf.metadata().ksName, cf.metadata().cfName)); + Set tableNames = new HashSet<>(); + for (PartitionUpdate update : updates) + tableNames.add(String.format("%s.%s", update.metadata().ksName, update.metadata().cfName)); String format = "Batch of prepared statements for {} is of size {}, exceeding specified threshold of {} by {}.{}"; if (size > failThreshold) { - Tracing.trace(format, ksCfPairs, size, failThreshold, size - failThreshold, " (see batch_size_fail_threshold_in_kb)"); - logger.error(format, ksCfPairs, size, failThreshold, size - failThreshold, " (see batch_size_fail_threshold_in_kb)"); + Tracing.trace(format, tableNames, size, failThreshold, size - failThreshold, " (see batch_size_fail_threshold_in_kb)"); + logger.error(format, tableNames, size, failThreshold, size - failThreshold, " (see batch_size_fail_threshold_in_kb)"); throw new InvalidRequestException("Batch too large"); } else if (logger.isWarnEnabled()) { - logger.warn(format, ksCfPairs, size, warnThreshold, size - warnThreshold, ""); + logger.warn(format, tableNames, size, warnThreshold, size - warnThreshold, ""); } - ClientWarn.warn(MessageFormatter.arrayFormat(format, new Object[] {ksCfPairs, size, warnThreshold, size - warnThreshold, ""}).getMessage()); + ClientWarn.warn(MessageFormatter.arrayFormat(format, new Object[] {tableNames, size, warnThreshold, size - warnThreshold, ""}).getMessage()); } } - private void verifyBatchType(Collection mutations) + private void verifyBatchType(Iterable updates) { - if (type != Type.LOGGED && mutations.size() > 1) + if (type != Type.LOGGED && Iterables.size(updates) > 1) { - Set ksCfPairs = new HashSet<>(); - Set keySet = new HashSet<>(); + Set keySet = new HashSet<>(); + Set tableNames = new HashSet<>(); - for (IMutation im : mutations) + for (PartitionUpdate update : updates) { - keySet.add(im.key()); - for (ColumnFamily cf : im.getColumnFamilies()) - ksCfPairs.add(String.format("%s.%s", cf.metadata().ksName, cf.metadata().cfName)); + keySet.add(update.partitionKey()); + tableNames.add(String.format("%s.%s", update.metadata().ksName, update.metadata().cfName)); } NoSpamLogger.log(logger, NoSpamLogger.Level.WARN, 1, TimeUnit.MINUTES, unloggedBatchWarning, keySet.size(), keySet.size() == 1 ? "" : "s", - ksCfPairs.size() == 1 ? "" : "s", ksCfPairs); + tableNames.size() == 1 ? "" : "s", tableNames); ClientWarn.warn(MessageFormatter.arrayFormat(unloggedBatchWarning, new Object[]{keySet.size(), keySet.size() == 1 ? "" : "s", - ksCfPairs.size() == 1 ? "" : "s", ksCfPairs}).getMessage()); + tableNames.size() == 1 ? "" : "s", tableNames}).getMessage()); } } @@ -326,17 +370,17 @@ public class BatchStatement implements CQLStatement private void executeWithoutConditions(Collection mutations, ConsistencyLevel cl) throws RequestExecutionException, RequestValidationException { - // Extract each collection of cfs from it's IMutation and then lazily concatenate all of them into a single Iterable. - Iterable cfs = Iterables.concat(Iterables.transform(mutations, new Function>() + // Extract each collection of updates from it's IMutation and then lazily concatenate all of them into a single Iterable. + Iterable updates = Iterables.concat(Iterables.transform(mutations, new Function>() { - public Collection apply(IMutation im) + public Collection apply(IMutation im) { - return im.getColumnFamilies(); + return im.getPartitionUpdates(); } })); - verifyBatchSize(cfs); - verifyBatchType(mutations); + verifyBatchSize(updates); + verifyBatchType(updates); boolean mutateAtomic = (type == Type.LOGGED && mutations.size() > 1); StorageProxy.mutateWithTriggers(mutations, cl, mutateAtomic); @@ -349,27 +393,26 @@ public class BatchStatement implements CQLStatement CQL3CasRequest casRequest = p.left; Set columnsWithConditions = p.right; - ColumnFamily result = StorageProxy.cas(casRequest.cfm.ksName, - casRequest.cfm.cfName, - casRequest.key, - casRequest, - options.getSerialConsistency(), - options.getConsistency(), - state.getClientState()); + String ksName = casRequest.cfm.ksName; + String tableName = casRequest.cfm.cfName; - return new ResultMessage.Rows(ModificationStatement.buildCasResultSet(casRequest.cfm.ksName, - casRequest.key, - casRequest.cfm.cfName, - result, - columnsWithConditions, - true, - options.forStatement(0))); + try (RowIterator result = StorageProxy.cas(ksName, + tableName, + casRequest.key, + casRequest, + options.getSerialConsistency(), + options.getConsistency(), + state.getClientState())) + { + return new ResultMessage.Rows(ModificationStatement.buildCasResultSet(ksName, tableName, result, columnsWithConditions, true, options.forStatement(0))); + } } + private Pair> makeCasRequest(BatchQueryOptions options, QueryState state) { long now = state.getTimestamp(); - ByteBuffer key = null; + DecoratedKey key = null; CQL3CasRequest casRequest = null; Set columnsWithConditions = new LinkedHashSet<>(); @@ -383,25 +426,25 @@ public class BatchStatement implements CQLStatement throw new IllegalArgumentException("Batch with conditions cannot span multiple partitions (you cannot use IN on the partition key)"); if (key == null) { - key = pks.get(0); - casRequest = new CQL3CasRequest(statement.cfm, key, true); + key = StorageService.getPartitioner().decorateKey(pks.get(0)); + casRequest = new CQL3CasRequest(statement.cfm, key, true, conditionColumns, updatesRegularRows, updatesStaticRow); } - else if (!key.equals(pks.get(0))) + else if (!key.getKey().equals(pks.get(0))) { throw new InvalidRequestException("Batch with conditions cannot span multiple partitions"); } - Composite clusteringPrefix = statement.createClusteringPrefix(statementOptions); + CBuilder cbuilder = statement.createClustering(statementOptions); if (statement.hasConditions()) { - statement.addConditions(clusteringPrefix, casRequest, statementOptions); + statement.addConditions(cbuilder.build(), casRequest, statementOptions); // As soon as we have a ifNotExists, we set columnsWithConditions to null so that everything is in the resultSet if (statement.hasIfNotExistCondition() || statement.hasIfExistCondition()) columnsWithConditions = null; else if (columnsWithConditions != null) Iterables.addAll(columnsWithConditions, statement.getColumnsWithConditions()); } - casRequest.addRowUpdate(clusteringPrefix, statement, statementOptions, timestamp); + casRequest.addRowUpdate(cbuilder, statement, statementOptions, timestamp); } return Pair.create(casRequest, columnsWithConditions); @@ -436,15 +479,13 @@ public class BatchStatement implements CQLStatement CQL3CasRequest request = p.left; Set columnsWithConditions = p.right; - ColumnFamily result = ModificationStatement.casInternal(request, state); + String ksName = request.cfm.ksName; + String tableName = request.cfm.cfName; - return new ResultMessage.Rows(ModificationStatement.buildCasResultSet(request.cfm.ksName, - request.key, - request.cfm.cfName, - result, - columnsWithConditions, - true, - options.forStatement(0))); + try (RowIterator result = ModificationStatement.casInternal(request, state)) + { + return new ResultMessage.Rows(ModificationStatement.buildCasResultSet(ksName, tableName, result, columnsWithConditions, true, options.forStatement(0))); + } } public interface BatchVariables diff --git a/src/java/org/apache/cassandra/cql3/statements/CQL3CasRequest.java b/src/java/org/apache/cassandra/cql3/statements/CQL3CasRequest.java index 081a14e9e3..9352930020 100644 --- a/src/java/org/apache/cassandra/cql3/statements/CQL3CasRequest.java +++ b/src/java/org/apache/cassandra/cql3/statements/CQL3CasRequest.java @@ -25,8 +25,8 @@ import com.google.common.collect.Multimap; import org.apache.cassandra.cql3.*; import org.apache.cassandra.config.CFMetaData; import org.apache.cassandra.db.*; -import org.apache.cassandra.db.composites.Composite; import org.apache.cassandra.db.filter.*; +import org.apache.cassandra.db.partitions.*; import org.apache.cassandra.exceptions.InvalidRequestException; import org.apache.cassandra.service.CASRequest; import org.apache.cassandra.utils.Pair; @@ -36,37 +36,45 @@ import org.apache.cassandra.utils.Pair; */ public class CQL3CasRequest implements CASRequest { - final CFMetaData cfm; - final ByteBuffer key; - final long now; - final boolean isBatch; + public final CFMetaData cfm; + public final DecoratedKey key; + public final boolean isBatch; + private final PartitionColumns conditionColumns; + private final boolean updatesRegularRows; + private final boolean updatesStaticRow; + private boolean hasExists; // whether we have an exist or if not exist condition - // We index RowCondition by the prefix of the row they applied to for 2 reasons: + // We index RowCondition by the clustering of the row they applied to for 2 reasons: // 1) this allows to keep things sorted to build the ColumnSlice array below // 2) this allows to detect when contradictory conditions are set (not exists with some other conditions on the same row) - private final SortedMap conditions; + private final SortedMap conditions; private final List updates = new ArrayList<>(); - public CQL3CasRequest(CFMetaData cfm, ByteBuffer key, boolean isBatch) + public CQL3CasRequest(CFMetaData cfm, + DecoratedKey key, + boolean isBatch, + PartitionColumns conditionColumns, + boolean updatesRegularRows, + boolean updatesStaticRow) { this.cfm = cfm; - // When checking if conditions apply, we want to use a fixed reference time for a whole request to check - // for expired cells. Note that this is unrelated to the cell timestamp. - this.now = System.currentTimeMillis(); this.key = key; this.conditions = new TreeMap<>(cfm.comparator); this.isBatch = isBatch; + this.conditionColumns = conditionColumns; + this.updatesRegularRows = updatesRegularRows; + this.updatesStaticRow = updatesStaticRow; } - public void addRowUpdate(Composite prefix, ModificationStatement stmt, QueryOptions options, long timestamp) + public void addRowUpdate(CBuilder cbuilder, ModificationStatement stmt, QueryOptions options, long timestamp) { - updates.add(new RowUpdate(prefix, stmt, options, timestamp)); + updates.add(new RowUpdate(cbuilder, stmt, options, timestamp)); } - public void addNotExist(Composite prefix) throws InvalidRequestException + public void addNotExist(Clustering clustering) throws InvalidRequestException { - RowCondition previous = conditions.put(prefix, new NotExistCondition(prefix, now)); + RowCondition previous = conditions.put(clustering, new NotExistCondition(clustering)); if (previous != null && !(previous instanceof NotExistCondition)) { // these should be prevented by the parser, but it doesn't hurt to check @@ -75,23 +83,25 @@ public class CQL3CasRequest implements CASRequest else throw new InvalidRequestException("Cannot mix IF conditions and IF NOT EXISTS for the same row"); } + hasExists = true; } - public void addExist(Composite prefix) throws InvalidRequestException + public void addExist(Clustering clustering) throws InvalidRequestException { - RowCondition previous = conditions.put(prefix, new ExistCondition(prefix, now)); + RowCondition previous = conditions.put(clustering, new ExistCondition(clustering)); // this should be prevented by the parser, but it doesn't hurt to check if (previous instanceof NotExistCondition) throw new InvalidRequestException("Cannot mix IF EXISTS and IF NOT EXISTS conditions for the same row"); + hasExists = true; } - public void addConditions(Composite prefix, Collection conds, QueryOptions options) throws InvalidRequestException + public void addConditions(Clustering clustering, Collection conds, QueryOptions options) throws InvalidRequestException { - RowCondition condition = conditions.get(prefix); + RowCondition condition = conditions.get(clustering); if (condition == null) { - condition = new ColumnsConditions(prefix, now); - conditions.put(prefix, condition); + condition = new ColumnsConditions(clustering); + conditions.put(clustering, condition); } else if (!(condition instanceof ColumnsConditions)) { @@ -100,24 +110,43 @@ public class CQL3CasRequest implements CASRequest ((ColumnsConditions)condition).addConditions(conds, options); } - public IDiskAtomFilter readFilter() + private PartitionColumns columnsToRead() + { + // If all our conditions are columns conditions (IF x = ?), then it's enough to query + // the columns from the conditions. If we have a IF EXISTS or IF NOT EXISTS however, + // we need to query all columns for the row since if the condition fails, we want to + // return everything to the user. Static columns make this a bit more complex, in that + // if an insert only static columns, then the existence condition applies only to the + // static columns themselves, and so we don't want to include regular columns in that + // case. + if (hasExists) + { + PartitionColumns allColumns = cfm.partitionColumns(); + Columns statics = updatesStaticRow ? allColumns.statics : Columns.NONE; + Columns regulars = updatesRegularRows ? allColumns.regulars : Columns.NONE; + return new PartitionColumns(statics, regulars); + } + return conditionColumns; + } + + public SinglePartitionReadCommand readCommand(int nowInSec) { assert !conditions.isEmpty(); - ColumnSlice[] slices = new ColumnSlice[conditions.size()]; - int i = 0; + Slices.Builder builder = new Slices.Builder(cfm.comparator, conditions.size()); // We always read CQL rows entirely as on CAS failure we want to be able to distinguish between "row exists // but all values for which there were conditions are null" and "row doesn't exists", and we can't rely on the // row marker for that (see #6623) - for (Composite prefix : conditions.keySet()) - slices[i++] = prefix.slice(); + for (Clustering clustering : conditions.keySet()) + { + if (clustering != Clustering.STATIC_CLUSTERING) + builder.add(Slice.make(clustering)); + } - int toGroup = cfm.comparator.isDense() ? -1 : cfm.clusteringColumns().size(); - slices = ColumnSlice.deoverlapSlices(slices, cfm.comparator); - assert ColumnSlice.validateSlices(slices, cfm.comparator, false); - return new SliceQueryFilter(slices, false, slices.length, toGroup); + ClusteringIndexSliceFilter filter = new ClusteringIndexSliceFilter(builder.build(), false); + return SinglePartitionReadCommand.create(cfm, nowInSec, key, ColumnFilter.selection(columnsToRead()), filter); } - public boolean appliesTo(ColumnFamily current) throws InvalidRequestException + public boolean appliesTo(FilteredPartition current) throws InvalidRequestException { for (RowCondition condition : conditions.values()) { @@ -127,16 +156,24 @@ public class CQL3CasRequest implements CASRequest return true; } - public ColumnFamily makeUpdates(ColumnFamily current) throws InvalidRequestException + private PartitionColumns updatedColumns() { - ColumnFamily cf = ArrayBackedSortedColumns.factory.create(cfm); + PartitionColumns.Builder builder = PartitionColumns.builder(); for (RowUpdate upd : updates) - upd.applyUpdates(current, cf); + builder.addAll(upd.stmt.updatedColumns()); + return builder.build(); + } + + public PartitionUpdate makeUpdates(FilteredPartition current) throws InvalidRequestException + { + PartitionUpdate update = new PartitionUpdate(cfm, key, updatedColumns(), conditions.size()); + for (RowUpdate upd : updates) + upd.applyUpdates(current, update); if (isBatch) - BatchStatement.verifyBatchSize(Collections.singleton(cf)); + BatchStatement.verifyBatchSize(Collections.singleton(update)); - return cf; + return update; } /** @@ -147,89 +184,62 @@ public class CQL3CasRequest implements CASRequest */ private class RowUpdate { - private final Composite rowPrefix; + private final CBuilder cbuilder; private final ModificationStatement stmt; private final QueryOptions options; private final long timestamp; - private RowUpdate(Composite rowPrefix, ModificationStatement stmt, QueryOptions options, long timestamp) + private RowUpdate(CBuilder cbuilder, ModificationStatement stmt, QueryOptions options, long timestamp) { - this.rowPrefix = rowPrefix; + this.cbuilder = cbuilder; this.stmt = stmt; this.options = options; this.timestamp = timestamp; } - public void applyUpdates(ColumnFamily current, ColumnFamily updates) throws InvalidRequestException + public void applyUpdates(FilteredPartition current, PartitionUpdate updates) throws InvalidRequestException { - Map map = null; - if (stmt.requiresRead()) - { - // Uses the "current" values read by Paxos for lists operation that requires a read - Iterator iter = cfm.comparator.CQL3RowBuilder(cfm, now).group(current.iterator(new ColumnSlice[]{ rowPrefix.slice() })); - if (iter.hasNext()) - { - map = Collections.singletonMap(key, iter.next()); - assert !iter.hasNext() : "We shoudn't be updating more than one CQL row per-ModificationStatement"; - } - } - - UpdateParameters params = new UpdateParameters(cfm, options, timestamp, stmt.getTimeToLive(options), map); - stmt.addUpdateForKey(updates, key, rowPrefix, params); + Map map = stmt.requiresRead() ? Collections.singletonMap(key, current) : null; + UpdateParameters params = new UpdateParameters(cfm, options, timestamp, stmt.getTimeToLive(options), map, true); + stmt.addUpdateForKey(updates, cbuilder, params); } } private static abstract class RowCondition { - public final Composite rowPrefix; - protected final long now; + public final Clustering clustering; - protected RowCondition(Composite rowPrefix, long now) + protected RowCondition(Clustering clustering) { - this.rowPrefix = rowPrefix; - this.now = now; + this.clustering = clustering; } - public abstract boolean appliesTo(ColumnFamily current) throws InvalidRequestException; + public abstract boolean appliesTo(FilteredPartition current) throws InvalidRequestException; } private static class NotExistCondition extends RowCondition { - private NotExistCondition(Composite rowPrefix, long now) + private NotExistCondition(Clustering clustering) { - super(rowPrefix, now); + super(clustering); } - public boolean appliesTo(ColumnFamily current) + public boolean appliesTo(FilteredPartition current) { - if (current == null) - return true; - - Iterator iter = current.iterator(new ColumnSlice[]{ rowPrefix.slice() }); - while (iter.hasNext()) - if (iter.next().isLive(now)) - return false; - return true; + return current == null || current.getRow(clustering) == null; } } private static class ExistCondition extends RowCondition { - private ExistCondition(Composite rowPrefix, long now) + private ExistCondition(Clustering clustering) { - super (rowPrefix, now); + super(clustering); } - public boolean appliesTo(ColumnFamily current) + public boolean appliesTo(FilteredPartition current) { - if (current == null) - return false; - - Iterator iter = current.iterator(new ColumnSlice[]{ rowPrefix.slice() }); - while (iter.hasNext()) - if (iter.next().isLive(now)) - return true; - return false; + return current != null && current.getRow(clustering) != null; } } @@ -237,9 +247,9 @@ public class CQL3CasRequest implements CASRequest { private final Multimap, ColumnCondition.Bound> conditions = HashMultimap.create(); - private ColumnsConditions(Composite rowPrefix, long now) + private ColumnsConditions(Clustering clustering) { - super(rowPrefix, now); + super(clustering); } public void addConditions(Collection conds, QueryOptions options) throws InvalidRequestException @@ -251,14 +261,16 @@ public class CQL3CasRequest implements CASRequest } } - public boolean appliesTo(ColumnFamily current) throws InvalidRequestException + public boolean appliesTo(FilteredPartition current) throws InvalidRequestException { if (current == null) return conditions.isEmpty(); for (ColumnCondition.Bound condition : conditions.values()) - if (!condition.appliesTo(rowPrefix, current, now)) + { + if (!condition.appliesTo(current.getRow(clustering))) return false; + } return true; } } diff --git a/src/java/org/apache/cassandra/cql3/statements/CreateIndexStatement.java b/src/java/org/apache/cassandra/cql3/statements/CreateIndexStatement.java index c3b0993006..cc808acc45 100644 --- a/src/java/org/apache/cassandra/cql3/statements/CreateIndexStatement.java +++ b/src/java/org/apache/cassandra/cql3/statements/CreateIndexStatement.java @@ -111,12 +111,14 @@ public class CreateIndexStatement extends SchemaAlteringStatement properties.validate(); - // TODO: we could lift that limitation - if ((cfm.comparator.isDense() || !cfm.comparator.isCompound()) && cd.isPrimaryKeyColumn()) - throw new InvalidRequestException("Secondary indexes are not supported on PRIMARY KEY columns in COMPACT STORAGE tables"); - - if (cd.kind == ColumnDefinition.Kind.COMPACT_VALUE) - throw new InvalidRequestException("Secondary indexes are not supported on COMPACT STORAGE tables that have clustering columns"); + if (cfm.isCompactTable()) + { + if (!cfm.isStaticCompactTable()) + throw new InvalidRequestException("Secondary indexes are not supported on COMPACT STORAGE tables that have clustering columns"); + else if (cd.isPrimaryKeyColumn()) + // TODO: we could lift that limitation + throw new InvalidRequestException("Secondary indexes are not supported on PRIMARY KEY columns in COMPACT STORAGE tables"); + } // It would be possible to support 2ndary index on static columns (but not without modifications of at least ExtendedFilter and // CompositesIndex) and maybe we should, but that means a query like: @@ -124,7 +126,7 @@ public class CreateIndexStatement extends SchemaAlteringStatement // would pull the full partition every time the static column of partition is 'bar', which sounds like offering a // fair potential for foot-shooting, so I prefer leaving that to a follow up ticket once we have identified cases where // such indexing is actually useful. - if (cd.isStatic()) + if (!cfm.isCompactTable() && cd.isStatic()) throw new InvalidRequestException("Secondary indexes are not allowed on static columns"); if (cd.kind == ColumnDefinition.Kind.PARTITION_KEY && cd.isOnAllComponents()) @@ -174,7 +176,7 @@ public class CreateIndexStatement extends SchemaAlteringStatement { cd.setIndexType(IndexType.CUSTOM, properties.getOptions()); } - else if (cfm.comparator.isCompound()) + else if (cfm.isCompound()) { Map options = Collections.emptyMap(); // For now, we only allow indexing values for collections, but we could later allow diff --git a/src/java/org/apache/cassandra/cql3/statements/CreateTableStatement.java b/src/java/org/apache/cassandra/cql3/statements/CreateTableStatement.java index b3591a22b4..8602af11a3 100644 --- a/src/java/org/apache/cassandra/cql3/statements/CreateTableStatement.java +++ b/src/java/org/apache/cassandra/cql3/statements/CreateTableStatement.java @@ -26,11 +26,8 @@ import org.apache.commons.lang3.StringUtils; import org.apache.cassandra.auth.*; import org.apache.cassandra.config.*; -import org.apache.cassandra.cql3.CFName; -import org.apache.cassandra.cql3.CQL3Type; -import org.apache.cassandra.cql3.ColumnIdentifier; -import org.apache.cassandra.db.ColumnFamilyType; -import org.apache.cassandra.db.composites.*; +import org.apache.cassandra.cql3.*; +import org.apache.cassandra.db.*; import org.apache.cassandra.db.marshal.*; import org.apache.cassandra.exceptions.*; import org.apache.cassandra.io.compress.CompressionParameters; @@ -43,15 +40,18 @@ import org.apache.cassandra.utils.ByteBufferUtil; /** A CREATE TABLE parsed from a CQL query statement. */ public class CreateTableStatement extends SchemaAlteringStatement { - public CellNameType comparator; - private AbstractType defaultValidator; - private AbstractType keyValidator; + private List> keyTypes; + private List> clusteringTypes; - private final List keyAliases = new ArrayList(); - private final List columnAliases = new ArrayList(); + private Map collections = new HashMap<>(); + + private final List keyAliases = new ArrayList<>(); + private final List columnAliases = new ArrayList<>(); private ByteBuffer valueAlias; private boolean isDense; + private boolean isCompound; + private boolean hasCounters; // use a TreeMap to preserve ordering across JDK versions (see CASSANDRA-9492) private final Map columns = new TreeMap<>(new Comparator() @@ -90,22 +90,6 @@ public class CreateTableStatement extends SchemaAlteringStatement // validated in announceMigration() } - // Column definitions - private List getColumns(CFMetaData cfm) - { - List columnDefs = new ArrayList<>(columns.size()); - Integer componentIndex = comparator.isCompound() ? comparator.clusteringPrefixSize() : null; - for (Map.Entry col : columns.entrySet()) - { - ColumnIdentifier id = col.getKey(); - columnDefs.add(staticColumns.contains(id) - ? ColumnDefinition.staticDef(cfm, col.getKey().bytes, col.getValue(), componentIndex) - : ColumnDefinition.regularDef(cfm, col.getKey().bytes, col.getValue(), componentIndex)); - } - - return columnDefs; - } - public boolean announceMigration(boolean isLocalOnly) throws RequestValidationException { try @@ -142,6 +126,46 @@ public class CreateTableStatement extends SchemaAlteringStatement } } + public CFMetaData.Builder metadataBuilder() + { + CFMetaData.Builder builder = CFMetaData.Builder.create(keyspace(), columnFamily(), isDense, isCompound, hasCounters); + for (int i = 0; i < keyAliases.size(); i++) + builder.addPartitionKey(keyAliases.get(i), keyTypes.get(i)); + for (int i = 0; i < columnAliases.size(); i++) + builder.addClusteringColumn(columnAliases.get(i), clusteringTypes.get(i)); + + boolean isStaticCompact = !isDense && !isCompound; + for (Map.Entry entry : columns.entrySet()) + { + ColumnIdentifier name = entry.getKey(); + // Note that for "static" no-clustering compact storage we use static for the defined columns + if (staticColumns.contains(name) || isStaticCompact) + builder.addStaticColumn(name, entry.getValue()); + else + builder.addRegularColumn(name, entry.getValue()); + } + + boolean isCompactTable = isDense || !isCompound; + if (isCompactTable) + { + CompactTables.DefaultNames names = CompactTables.defaultNameGenerator(builder.usedColumnNames()); + // Compact tables always have a clustering and a single regular value. + if (isStaticCompact) + { + builder.addClusteringColumn(names.defaultClusteringName(), UTF8Type.instance); + builder.addRegularColumn(names.defaultCompactValueName(), hasCounters ? CounterColumnType.instance : BytesType.instance); + } + else if (isDense && !builder.hasRegulars()) + { + // Even for dense, we might not have our regular column if it wasn't part of the declaration. If + // that's the case, add it but with a specific EmptyType so we can recognize that case later + builder.addRegularColumn(names.defaultCompactValueName(), EmptyType.instance); + } + } + + return builder; + } + /** * Returns a CFMetaData instance based on the parameters parsed from this * CREATE statement, or defaults where applicable. @@ -151,48 +175,16 @@ public class CreateTableStatement extends SchemaAlteringStatement */ public CFMetaData getCFMetaData() throws RequestValidationException { - CFMetaData newCFMD; - newCFMD = new CFMetaData(keyspace(), - columnFamily(), - ColumnFamilyType.Standard, - comparator); + CFMetaData newCFMD = metadataBuilder().build(); applyPropertiesTo(newCFMD); return newCFMD; } public void applyPropertiesTo(CFMetaData cfmd) throws RequestValidationException { - cfmd.defaultValidator(defaultValidator) - .keyValidator(keyValidator) - .addAllColumnDefinitions(getColumns(cfmd)) - .isDense(isDense); - - addColumnMetadataFromAliases(cfmd, keyAliases, keyValidator, ColumnDefinition.Kind.PARTITION_KEY); - addColumnMetadataFromAliases(cfmd, columnAliases, comparator.asAbstractType(), ColumnDefinition.Kind.CLUSTERING_COLUMN); - if (valueAlias != null) - addColumnMetadataFromAliases(cfmd, Collections.singletonList(valueAlias), defaultValidator, ColumnDefinition.Kind.COMPACT_VALUE); - properties.applyToCFMetadata(cfmd); } - private void addColumnMetadataFromAliases(CFMetaData cfm, List aliases, AbstractType comparator, ColumnDefinition.Kind kind) - { - if (comparator instanceof CompositeType) - { - CompositeType ct = (CompositeType)comparator; - for (int i = 0; i < aliases.size(); ++i) - if (aliases.get(i) != null) - cfm.addOrReplaceColumnDefinition(new ColumnDefinition(cfm, aliases.get(i), ct.types.get(i), i, kind)); - } - else - { - assert aliases.size() <= 1; - if (!aliases.isEmpty() && aliases.get(0) != null) - cfm.addOrReplaceColumnDefinition(new ColumnDefinition(cfm, aliases.get(0), comparator, null, kind)); - } - } - - public static class RawStatement extends CFStatement { private final Map definitions = new HashMap<>(); @@ -233,169 +225,107 @@ public class CreateTableStatement extends SchemaAlteringStatement CreateTableStatement stmt = new CreateTableStatement(cfName, properties, ifNotExists, staticColumns); - boolean hasCounters = false; - Map definedMultiCellCollections = null; for (Map.Entry entry : definitions.entrySet()) { ColumnIdentifier id = entry.getKey(); CQL3Type pt = entry.getValue().prepare(keyspace()); - if (pt.isCollection() && ((CollectionType) pt.getType()).isMultiCell()) - { - if (definedMultiCellCollections == null) - definedMultiCellCollections = new HashMap<>(); - definedMultiCellCollections.put(id.bytes, (CollectionType) pt.getType()); - } - else if (entry.getValue().isCounter()) - hasCounters = true; - + if (pt.isCollection() && ((CollectionType)pt.getType()).isMultiCell()) + stmt.collections.put(id.bytes, (CollectionType)pt.getType()); + if (entry.getValue().isCounter()) + stmt.hasCounters = true; stmt.columns.put(id, pt.getType()); // we'll remove what is not a column below } if (keyAliases.isEmpty()) throw new InvalidRequestException("No PRIMARY KEY specifed (exactly one required)"); - else if (keyAliases.size() > 1) + if (keyAliases.size() > 1) throw new InvalidRequestException("Multiple PRIMARY KEYs specifed (exactly one required)"); - else if (hasCounters && properties.getDefaultTimeToLive() > 0) + if (stmt.hasCounters && properties.getDefaultTimeToLive() > 0) throw new InvalidRequestException("Cannot set default_time_to_live on a table with counters"); List kAliases = keyAliases.get(0); - - List> keyTypes = new ArrayList>(kAliases.size()); + stmt.keyTypes = new ArrayList>(kAliases.size()); for (ColumnIdentifier alias : kAliases) { - stmt.keyAliases.add(alias.bytes); + stmt.keyAliases.add(alias); AbstractType t = getTypeAndRemove(stmt.columns, alias); if (t instanceof CounterColumnType) throw new InvalidRequestException(String.format("counter type is not supported for PRIMARY KEY part %s", alias)); if (staticColumns.contains(alias)) throw new InvalidRequestException(String.format("Static column %s cannot be part of the PRIMARY KEY", alias)); - keyTypes.add(t); + stmt.keyTypes.add(t); } - stmt.keyValidator = keyTypes.size() == 1 ? keyTypes.get(0) : CompositeType.getInstance(keyTypes); - - // Dense means that no part of the comparator stores a CQL column name. This means - // COMPACT STORAGE with at least one columnAliases (otherwise it's a thrift "static" CF). - stmt.isDense = useCompactStorage && !columnAliases.isEmpty(); + stmt.clusteringTypes = new ArrayList<>(columnAliases.size()); // Handle column aliases - if (columnAliases.isEmpty()) + for (ColumnIdentifier t : columnAliases) { - if (useCompactStorage) + stmt.columnAliases.add(t); + + AbstractType type = getTypeAndRemove(stmt.columns, t); + if (type instanceof CounterColumnType) + throw new InvalidRequestException(String.format("counter type is not supported for PRIMARY KEY part %s", t)); + if (staticColumns.contains(t)) + throw new InvalidRequestException(String.format("Static column %s cannot be part of the PRIMARY KEY", t)); + stmt.clusteringTypes.add(type); + } + + // We've handled anything that is not a rpimary key so stmt.columns only contains NON-PK columns. So + // if it's a counter table, make sure we don't have non-counter types + if (stmt.hasCounters) + { + for (AbstractType type : stmt.columns.values()) + if (!type.isCounter()) + throw new InvalidRequestException("Cannot mix counter and non counter columns in the same table"); + } + + // Dense means that on the thrift side, no part of the "thrift column name" stores a "CQL/metadata column name". + // This means COMPACT STORAGE with at least one clustering type (otherwise it's a thrift "static" CF). + stmt.isDense = useCompactStorage && !stmt.clusteringTypes.isEmpty(); + // Compound means that on the thrift side, the "thrift column name" is a composite one. It's the case unless + // we use compact storage COMPACT STORAGE and we have either no clustering columns (thrift "static" CF) or + // only one of them (if more than one, it's a "dense composite"). + stmt.isCompound = !(useCompactStorage && stmt.clusteringTypes.size() <= 1); + + // For COMPACT STORAGE, we reject any "feature" that we wouldn't be able to translate back to thrift. + if (useCompactStorage) + { + if (!stmt.collections.isEmpty()) + throw new InvalidRequestException("Non-frozen collection types are not supported with COMPACT STORAGE"); + if (!staticColumns.isEmpty()) + throw new InvalidRequestException("Static columns are not supported in COMPACT STORAGE tables"); + + if (stmt.clusteringTypes.isEmpty()) { - // There should remain some column definition since it is a non-composite "static" CF + // It's a thrift "static CF" so there should be some columns definition if (stmt.columns.isEmpty()) throw new InvalidRequestException("No definition found that is not part of the PRIMARY KEY"); - - if (definedMultiCellCollections != null) - throw new InvalidRequestException("Non-frozen collection types are not supported with COMPACT STORAGE"); - - stmt.comparator = new SimpleSparseCellNameType(UTF8Type.instance); } - else - { - stmt.comparator = definedMultiCellCollections == null - ? new CompoundSparseCellNameType(Collections.>emptyList()) - : new CompoundSparseCellNameType.WithCollection(Collections.>emptyList(), ColumnToCollectionType.getInstance(definedMultiCellCollections)); - } - } - else - { - // If we use compact storage and have only one alias, it is a - // standard "dynamic" CF, otherwise it's a composite - if (useCompactStorage && columnAliases.size() == 1) - { - if (definedMultiCellCollections != null) - throw new InvalidRequestException("Collection types are not supported with COMPACT STORAGE"); - - ColumnIdentifier alias = columnAliases.get(0); - if (staticColumns.contains(alias)) - throw new InvalidRequestException(String.format("Static column %s cannot be part of the PRIMARY KEY", alias)); - - stmt.columnAliases.add(alias.bytes); - AbstractType at = getTypeAndRemove(stmt.columns, alias); - if (at instanceof CounterColumnType) - throw new InvalidRequestException(String.format("counter type is not supported for PRIMARY KEY part %s", stmt.columnAliases.get(0))); - stmt.comparator = new SimpleDenseCellNameType(at); - } - else - { - List> types = new ArrayList>(columnAliases.size() + 1); - for (ColumnIdentifier t : columnAliases) - { - stmt.columnAliases.add(t.bytes); - - AbstractType type = getTypeAndRemove(stmt.columns, t); - if (type instanceof CounterColumnType) - throw new InvalidRequestException(String.format("counter type is not supported for PRIMARY KEY part %s", t)); - if (staticColumns.contains(t)) - throw new InvalidRequestException(String.format("Static column %s cannot be part of the PRIMARY KEY", t)); - types.add(type); - } - - if (useCompactStorage) - { - if (definedMultiCellCollections != null) - throw new InvalidRequestException("Collection types are not supported with COMPACT STORAGE"); - - stmt.comparator = new CompoundDenseCellNameType(types); - } - else - { - stmt.comparator = definedMultiCellCollections == null - ? new CompoundSparseCellNameType(types) - : new CompoundSparseCellNameType.WithCollection(types, ColumnToCollectionType.getInstance(definedMultiCellCollections)); - } - } - } - - if (!staticColumns.isEmpty()) - { - // Only CQL3 tables can have static columns - if (useCompactStorage) - throw new InvalidRequestException("Static columns are not supported in COMPACT STORAGE tables"); - // Static columns only make sense if we have at least one clustering column. Otherwise everything is static anyway - if (columnAliases.isEmpty()) - throw new InvalidRequestException("Static columns are only useful (and thus allowed) if the table has at least one clustering column"); - } - - if (useCompactStorage && !stmt.columnAliases.isEmpty()) - { - if (stmt.columns.isEmpty()) - { - // The only value we'll insert will be the empty one, so the default validator don't matter - stmt.defaultValidator = BytesType.instance; - // We need to distinguish between - // * I'm upgrading from thrift so the valueAlias is null - // * I've defined my table with only a PK (and the column value will be empty) - // So, we use an empty valueAlias (rather than null) for the second case - stmt.valueAlias = ByteBufferUtil.EMPTY_BYTE_BUFFER; - } - else + + if (stmt.isDense) { + // We can have no columns (only the PK), but we can't have more than one. if (stmt.columns.size() > 1) throw new InvalidRequestException(String.format("COMPACT STORAGE with composite PRIMARY KEY allows no more than one column not part of the PRIMARY KEY (got: %s)", StringUtils.join(stmt.columns.keySet(), ", "))); - - Map.Entry lastEntry = stmt.columns.entrySet().iterator().next(); - stmt.defaultValidator = lastEntry.getValue(); - stmt.valueAlias = lastEntry.getKey().bytes; - stmt.columns.remove(lastEntry.getKey()); + } + else + { + // we are in the "static" case, so we need at least one column defined. For non-compact however, having + // just the PK is fine. + if (stmt.columns.isEmpty()) + throw new InvalidRequestException("COMPACT STORAGE with non-composite PRIMARY KEY require one column not part of the PRIMARY KEY, none given"); } } else { - // For compact, we are in the "static" case, so we need at least one column defined. For non-compact however, having - // just the PK is fine since we have CQL3 row marker. - if (useCompactStorage && stmt.columns.isEmpty()) - throw new InvalidRequestException("COMPACT STORAGE with non-composite PRIMARY KEY require one column not part of the PRIMARY KEY, none given"); - - // There is no way to insert/access a column that is not defined for non-compact storage, so - // the actual validator don't matter much (except that we want to recognize counter CF as limitation apply to them). - stmt.defaultValidator = !stmt.columns.isEmpty() && (stmt.columns.values().iterator().next() instanceof CounterColumnType) - ? CounterColumnType.instance - : BytesType.instance; + if (stmt.clusteringTypes.isEmpty() && !staticColumns.isEmpty()) + { + // Static columns only make sense if we have at least one clustering column. Otherwise everything is static anyway + if (columnAliases.isEmpty()) + throw new InvalidRequestException("Static columns are only useful (and thus allowed) if the table has at least one clustering column"); + } } - // If we give a clustering order, we must explicitly do so for all aliases and in the order of the PK if (!definedOrdering.isEmpty()) { diff --git a/src/java/org/apache/cassandra/cql3/statements/DeleteStatement.java b/src/java/org/apache/cassandra/cql3/statements/DeleteStatement.java index ff685cfde4..b4d7853852 100644 --- a/src/java/org/apache/cassandra/cql3/statements/DeleteStatement.java +++ b/src/java/org/apache/cassandra/cql3/statements/DeleteStatement.java @@ -17,7 +17,6 @@ */ package org.apache.cassandra.cql3.statements; -import java.nio.ByteBuffer; import java.util.*; import com.google.common.collect.Iterators; @@ -27,7 +26,8 @@ import org.apache.cassandra.cql3.restrictions.Restriction; import org.apache.cassandra.config.CFMetaData; import org.apache.cassandra.config.ColumnDefinition; import org.apache.cassandra.db.*; -import org.apache.cassandra.db.composites.Composite; +import org.apache.cassandra.db.rows.Row; +import org.apache.cassandra.db.partitions.*; import org.apache.cassandra.exceptions.*; import org.apache.cassandra.utils.Pair; @@ -46,44 +46,57 @@ public class DeleteStatement extends ModificationStatement return false; } - public void addUpdateForKey(ColumnFamily cf, ByteBuffer key, Composite prefix, UpdateParameters params) + public void addUpdateForKey(PartitionUpdate update, CBuilder cbuilder, UpdateParameters params) throws InvalidRequestException { - List deletions = getOperations(); + List regularDeletions = getRegularOperations(); + List staticDeletions = getStaticOperations(); - if (prefix.size() < cfm.clusteringColumns().size() && !deletions.isEmpty()) + if (regularDeletions.isEmpty() && staticDeletions.isEmpty()) { - // In general, we can't delete specific columns if not all clustering columns have been specified. - // However, if we delete only static colums, it's fine since we won't really use the prefix anyway. - for (Operation deletion : deletions) - if (!deletion.column.isStatic()) - throw new InvalidRequestException(String.format("Primary key column '%s' must be specified in order to delete column '%s'", getFirstEmptyKey().name, deletion.column.name)); - } - - if (deletions.isEmpty()) - { - // We delete the slice selected by the prefix. - // However, for performance reasons, we distinguish 2 cases: - // - It's a full internal row delete - // - It's a full cell name (i.e it's a dense layout and the prefix is full) - if (prefix.isEmpty()) + // We're not deleting any specific columns so it's either a full partition deletion .... + if (cbuilder.count() == 0) { - // No columns specified, delete the row - cf.delete(new DeletionInfo(params.timestamp, params.localDeletionTime)); + update.addPartitionDeletion(params.deletionTime()); } - else if (cfm.comparator.isDense() && prefix.size() == cfm.clusteringColumns().size()) + // ... or a row deletion ... + else if (cbuilder.remainingCount() == 0) { - cf.addAtom(params.makeTombstone(cfm.comparator.create(prefix, null))); + Clustering clustering = cbuilder.build(); + Row.Writer writer = update.writer(); + params.writeClustering(clustering, writer); + params.writeRowDeletion(writer); + writer.endOfRow(); } + // ... or a range of rows deletion. else { - cf.addAtom(params.makeRangeTombstone(prefix.slice())); + update.addRangeTombstone(params.makeRangeTombstone(cbuilder)); } } else { - for (Operation op : deletions) - op.execute(key, cf, prefix, params); + if (!regularDeletions.isEmpty()) + { + // We can't delete specific (regular) columns if not all clustering columns have been specified. + if (cbuilder.remainingCount() > 0) + throw new InvalidRequestException(String.format("Primary key column '%s' must be specified in order to delete column '%s'", getFirstEmptyKey().name, regularDeletions.get(0).column.name)); + + Clustering clustering = cbuilder.build(); + Row.Writer writer = update.writer(); + params.writeClustering(clustering, writer); + for (Operation op : regularDeletions) + op.execute(update.partitionKey(), clustering, writer, params); + writer.endOfRow(); + } + + if (!staticDeletions.isEmpty()) + { + Row.Writer writer = update.staticWriter(); + for (Operation op : staticDeletions) + op.execute(update.partitionKey(), Clustering.STATIC_CLUSTERING, writer, params); + writer.endOfRow(); + } } } diff --git a/src/java/org/apache/cassandra/cql3/statements/DropTypeStatement.java b/src/java/org/apache/cassandra/cql3/statements/DropTypeStatement.java index ed10d00a68..8ad4f6ca34 100644 --- a/src/java/org/apache/cassandra/cql3/statements/DropTypeStatement.java +++ b/src/java/org/apache/cassandra/cql3/statements/DropTypeStatement.java @@ -119,12 +119,6 @@ public class DropTypeStatement extends SchemaAlteringStatement if (isUsedBy(subtype)) return true; } - else if (toCheck instanceof ColumnToCollectionType) - { - for (CollectionType collection : ((ColumnToCollectionType)toCheck).defined.values()) - if (isUsedBy(collection)) - return true; - } else if (toCheck instanceof CollectionType) { if (toCheck instanceof ListType) diff --git a/src/java/org/apache/cassandra/cql3/statements/ModificationStatement.java b/src/java/org/apache/cassandra/cql3/statements/ModificationStatement.java index 888cdb77f3..6a6d18670d 100644 --- a/src/java/org/apache/cassandra/cql3/statements/ModificationStatement.java +++ b/src/java/org/apache/cassandra/cql3/statements/ModificationStatement.java @@ -21,7 +21,8 @@ import java.nio.ByteBuffer; import java.util.*; import com.google.common.collect.Iterables; -import com.google.common.collect.Lists; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; import org.apache.cassandra.auth.Permission; import org.apache.cassandra.config.CFMetaData; @@ -33,19 +34,20 @@ import org.apache.cassandra.cql3.restrictions.Restriction; import org.apache.cassandra.cql3.restrictions.SingleColumnRestriction; import org.apache.cassandra.cql3.selection.Selection; import org.apache.cassandra.db.*; -import org.apache.cassandra.db.composites.Composite; -import org.apache.cassandra.db.composites.CompositesBuilder; -import org.apache.cassandra.db.filter.ColumnSlice; -import org.apache.cassandra.db.filter.SliceQueryFilter; +import org.apache.cassandra.db.rows.*; +import org.apache.cassandra.db.partitions.*; +import org.apache.cassandra.db.filter.*; import org.apache.cassandra.db.marshal.BooleanType; import org.apache.cassandra.exceptions.*; import org.apache.cassandra.service.ClientState; import org.apache.cassandra.service.QueryState; import org.apache.cassandra.service.StorageProxy; +import org.apache.cassandra.service.StorageService; import org.apache.cassandra.service.paxos.Commit; import org.apache.cassandra.thrift.ThriftValidation; import org.apache.cassandra.transport.messages.ResultMessage; import org.apache.cassandra.triggers.TriggerExecutor; +import org.apache.cassandra.utils.FBUtilities; import org.apache.cassandra.utils.Pair; import org.apache.cassandra.utils.UUIDGen; @@ -58,6 +60,8 @@ import static org.apache.cassandra.cql3.statements.RequestValidations.invalidReq */ public abstract class ModificationStatement implements CQLStatement { + protected static final Logger logger = LoggerFactory.getLogger(ModificationStatement.class); + private static final ColumnIdentifier CAS_RESULT_COLUMN = new ColumnIdentifier("[applied]", false); public static enum StatementType { INSERT, UPDATE, DELETE } @@ -68,7 +72,15 @@ public abstract class ModificationStatement implements CQLStatement public final Attributes attrs; protected final Map processedKeys = new HashMap<>(); - private final List columnOperations = new ArrayList(); + private final List regularOperations = new ArrayList<>(); + private final List staticOperations = new ArrayList<>(); + + // TODO: If we had a builder for this statement, we could have updatedColumns/conditionColumns final and only have + // updatedColumnsBuilder/conditionColumnsBuilder in the builder ... + private PartitionColumns updatedColumns; + private PartitionColumns.Builder updatedColumnsBuilder = PartitionColumns.builder(); + private PartitionColumns conditionColumns; + private PartitionColumns.Builder conditionColumnsBuilder = PartitionColumns.builder(); // Separating normal and static conditions makes things somewhat easier private List columnConditions; @@ -103,25 +115,24 @@ public abstract class ModificationStatement implements CQLStatement Iterable functions = attrs.getFunctions(); for (Restriction restriction : processedKeys.values()) - functions = Iterables.concat(functions, restriction.getFunctions()); + functions = Iterables.concat(functions, restriction.getFunctions()); - if (columnOperations != null) - for (Operation operation : columnOperations) - functions = Iterables.concat(functions, operation.getFunctions()); + for (Operation operation : allOperations()) + functions = Iterables.concat(functions, operation.getFunctions()); - if (columnConditions != null) - for (ColumnCondition condition : columnConditions) - functions = Iterables.concat(functions, condition.getFunctions()); - - if (staticConditions != null) - for (ColumnCondition condition : staticConditions) - functions = Iterables.concat(functions, condition.getFunctions()); + for (ColumnCondition condition : allConditions()) + functions = Iterables.concat(functions, condition.getFunctions()); return functions; } + public boolean hasNoClusteringColumns() + { + return hasNoClusteringColumns; + } + public abstract boolean requireFullClusteringKey(); - public abstract void addUpdateForKey(ColumnFamily updates, ByteBuffer key, Composite prefix, UpdateParameters params) throws InvalidRequestException; + public abstract void addUpdateForKey(PartitionUpdate update, CBuilder clusteringBuilder, UpdateParameters params) throws InvalidRequestException; public int getBoundTerms() { @@ -184,16 +195,75 @@ public abstract class ModificationStatement implements CQLStatement public void addOperation(Operation op) { + updatedColumnsBuilder.add(op.column); + // If the operation requires a read-before-write and we're doing a conditional read, we want to read + // the affected column as part of the read-for-conditions paxos phase (see #7499). + if (op.requiresRead()) + conditionColumnsBuilder.add(op.column); + if (op.column.isStatic()) + { setsStaticColumns = true; + staticOperations.add(op); + } else + { setsRegularColumns = true; - columnOperations.add(op); + regularOperations.add(op); + } } - public List getOperations() + public PartitionColumns updatedColumns() { - return columnOperations; + return updatedColumns; + } + + public PartitionColumns conditionColumns() + { + return conditionColumns; + } + + public boolean updatesRegularRows() + { + // We're updating regular rows if all the clustering columns are provided. + // Note that the only case where we're allowed not to provide clustering + // columns is if we set some static columns, and in that case no clustering + // columns should be given. So in practice, it's enough to check if we have + // either the table has no clustering or if it has at least one of them set. + return cfm.clusteringColumns().isEmpty() || !hasNoClusteringColumns; + } + + public boolean updatesStaticRow() + { + return !staticOperations.isEmpty(); + } + + private void finishPreparation() + { + updatedColumns = updatedColumnsBuilder.build(); + // Compact tables have not row marker. So if we don't actually update any particular column, + // this means that we're only updating the PK, which we allow if only those were declared in + // the definition. In that case however, we do went to write the compactValueColumn (since again + // we can't use a "row marker") so add it automatically. + if (cfm.isCompactTable() && updatedColumns.isEmpty() && updatesRegularRows()) + updatedColumns = cfm.partitionColumns(); + + conditionColumns = conditionColumnsBuilder.build(); + } + + public List getRegularOperations() + { + return regularOperations; + } + + public List getStaticOperations() + { + return staticOperations; + } + + public Iterable allOperations() + { + return Iterables.concat(staticOperations, regularOperations); } public Iterable getColumnsWithConditions() @@ -205,8 +275,19 @@ public abstract class ModificationStatement implements CQLStatement staticConditions == null ? Collections.emptyList() : Iterables.transform(staticConditions, getColumnForCondition)); } + public Iterable allConditions() + { + if (staticConditions == null) + return columnConditions == null ? Collections.emptySet(): columnConditions; + if (columnConditions == null) + return staticConditions; + return Iterables.concat(staticConditions, columnConditions); + } + public void addCondition(ColumnCondition cond) { + conditionColumnsBuilder.add(cond.column); + List conds = null; if (cond.column.isStatic()) { @@ -255,7 +336,7 @@ public abstract class ModificationStatement implements CQLStatement public void addKeyValue(ColumnDefinition def, Term value) throws InvalidRequestException { - addKeyValues(def, new SingleColumnRestriction.EQ(def, value)); + addKeyValues(def, new SingleColumnRestriction.EQRestriction(def, value)); } public void processWhereClause(List whereClause, VariableSpecifications names) throws InvalidRequestException @@ -303,26 +384,25 @@ public abstract class ModificationStatement implements CQLStatement public List buildPartitionKeyNames(QueryOptions options) throws InvalidRequestException { - CompositesBuilder keyBuilder = new CompositesBuilder(cfm.getKeyValidatorAsCType()); + MultiCBuilder keyBuilder = MultiCBuilder.create(cfm.getKeyValidatorAsClusteringComparator()); for (ColumnDefinition def : cfm.partitionKeyColumns()) { Restriction r = checkNotNull(processedKeys.get(def.name), "Missing mandatory PRIMARY KEY part %s", def.name); r.appendTo(keyBuilder, options); } - return Lists.transform(keyBuilder.build(), new com.google.common.base.Function() + NavigableSet clusterings = keyBuilder.build(); + List keys = new ArrayList(clusterings.size()); + for (Clustering clustering : clusterings) { - @Override - public ByteBuffer apply(Composite composite) - { - ByteBuffer byteBuffer = composite.toByteBuffer(); - ThriftValidation.validateKey(cfm, byteBuffer); - return byteBuffer; - } - }); + ByteBuffer key = CFMetaData.serializePartitionKey(clustering); + ThriftValidation.validateKey(cfm, key); + keys.add(key); + } + return keys; } - public Composite createClusteringPrefix(QueryOptions options) + public CBuilder createClustering(QueryOptions options) throws InvalidRequestException { // If the only updated/deleted columns are static, then we don't need clustering columns. @@ -339,7 +419,7 @@ public abstract class ModificationStatement implements CQLStatement { // If we set no non-static columns, then it's fine not to have clustering columns if (hasNoClusteringColumns) - return cfm.comparator.staticPrefix(); + return CBuilder.STATIC_BUILDER; // If we do have clustering columns however, then either it's an INSERT and the query is valid // but we still need to build a proper prefix, or it's not an INSERT, and then we want to reject @@ -354,13 +434,15 @@ public abstract class ModificationStatement implements CQLStatement } } - return createClusteringPrefixBuilderInternal(options); + return createClusteringInternal(options); } - private Composite createClusteringPrefixBuilderInternal(QueryOptions options) + private CBuilder createClusteringInternal(QueryOptions options) throws InvalidRequestException { - CompositesBuilder builder = new CompositesBuilder(cfm.comparator); + CBuilder builder = CBuilder.create(cfm.comparator); + MultiCBuilder multiBuilder = MultiCBuilder.wrap(builder); + ColumnDefinition firstEmptyKey = null; for (ColumnDefinition def : cfm.clusteringColumns()) { @@ -368,7 +450,7 @@ public abstract class ModificationStatement implements CQLStatement if (r == null) { firstEmptyKey = def; - checkFalse(requireFullClusteringKey() && !cfm.comparator.isDense() && cfm.comparator.isCompound(), + checkFalse(requireFullClusteringKey() && !cfm.isDense() && cfm.isCompound(), "Missing mandatory PRIMARY KEY part %s", def.name); } else if (firstEmptyKey != null) @@ -377,10 +459,10 @@ public abstract class ModificationStatement implements CQLStatement } else { - r.appendTo(builder, options); + r.appendTo(multiBuilder, options); } } - return builder.build().get(0); // We only allow IN for row keys so far + return builder; } protected ColumnDefinition getFirstEmptyKey() @@ -396,14 +478,14 @@ public abstract class ModificationStatement implements CQLStatement public boolean requiresRead() { // Lists SET operation incurs a read. - for (Operation op : columnOperations) + for (Operation op : allOperations()) if (op.requiresRead()) return true; return false; } - protected Map readRequiredRows(Collection partitionKeys, Composite clusteringPrefix, boolean local, ConsistencyLevel cl) + protected Map readRequiredLists(Collection partitionKeys, CBuilder cbuilder, boolean local, ConsistencyLevel cl) throws RequestExecutionException, RequestValidationException { if (!requiresRead()) @@ -418,32 +500,54 @@ public abstract class ModificationStatement implements CQLStatement throw new InvalidRequestException(String.format("Write operation require a read but consistency %s is not supported on reads", cl)); } - ColumnSlice[] slices = new ColumnSlice[]{ clusteringPrefix.slice() }; - List commands = new ArrayList(partitionKeys.size()); - long now = System.currentTimeMillis(); + // TODO: no point in recomputing that every time. Should move to preparation phase. + PartitionColumns.Builder builder = PartitionColumns.builder(); + for (Operation op : allOperations()) + if (op.requiresRead()) + builder.add(op.column); + + PartitionColumns toRead = builder.build(); + + NavigableSet clusterings = FBUtilities.singleton(cbuilder.build(), cfm.comparator); + List> commands = new ArrayList<>(partitionKeys.size()); + int nowInSec = FBUtilities.nowInSeconds(); for (ByteBuffer key : partitionKeys) - commands.add(new SliceFromReadCommand(keyspace(), - key, - columnFamily(), - now, - new SliceQueryFilter(slices, false, Integer.MAX_VALUE))); + commands.add(new SinglePartitionNamesCommand(cfm, + nowInSec, + ColumnFilter.selection(toRead), + RowFilter.NONE, + DataLimits.NONE, + StorageService.getPartitioner().decorateKey(key), + new ClusteringIndexNamesFilter(clusterings, false))); - List rows = local - ? SelectStatement.readLocally(keyspace(), commands) - : StorageProxy.read(commands, cl); + Map map = new HashMap(); - Map map = new HashMap(); - for (Row row : rows) + SinglePartitionReadCommand.Group group = new SinglePartitionReadCommand.Group(commands, DataLimits.NONE); + + if (local) { - if (row.cf == null || row.cf.isEmpty()) - continue; - - Iterator iter = cfm.comparator.CQL3RowBuilder(cfm, now).group(row.cf.getSortedColumns().iterator()); - if (iter.hasNext()) + try (ReadOrderGroup orderGroup = group.startOrderGroup(); PartitionIterator iter = group.executeInternal(orderGroup)) { - map.put(row.key.getKey(), iter.next()); - // We can only update one CQ3Row per partition key at a time (we don't allow IN for clustering key) - assert !iter.hasNext(); + return asMaterializedMap(iter); + } + } + else + { + try (PartitionIterator iter = group.execute(cl, null)) + { + return asMaterializedMap(iter); + } + } + } + + private Map asMaterializedMap(PartitionIterator iterator) + { + Map map = new HashMap(); + while (iterator.hasNext()) + { + try (RowIterator partition = iterator.next()) + { + map.put(partition.partitionKey(), FilteredPartition.create(partition)); } } return map; @@ -492,14 +596,16 @@ public abstract class ModificationStatement implements CQLStatement { CQL3CasRequest request = makeCasRequest(queryState, options); - ColumnFamily result = StorageProxy.cas(keyspace(), - columnFamily(), - request.key, - request, - options.getSerialConsistency(), - options.getConsistency(), - queryState.getClientState()); - return new ResultMessage.Rows(buildCasResultSet(request.key, result, options)); + try (RowIterator result = StorageProxy.cas(keyspace(), + columnFamily(), + request.key, + request, + options.getSerialConsistency(), + options.getConsistency(), + queryState.getClientState())) + { + return new ResultMessage.Rows(buildCasResultSet(result, options)); + } } private CQL3CasRequest makeCasRequest(QueryState queryState, QueryOptions options) @@ -509,54 +615,54 @@ public abstract class ModificationStatement implements CQLStatement if (keys.size() > 1) throw new InvalidRequestException("IN on the partition key is not supported with conditional updates"); - ByteBuffer key = keys.get(0); + DecoratedKey key = StorageService.getPartitioner().decorateKey(keys.get(0)); long now = options.getTimestamp(queryState); - Composite prefix = createClusteringPrefix(options); + CBuilder cbuilder = createClustering(options); - CQL3CasRequest request = new CQL3CasRequest(cfm, key, false); - addConditions(prefix, request, options); - request.addRowUpdate(prefix, this, options, now); + CQL3CasRequest request = new CQL3CasRequest(cfm, key, false, conditionColumns(), updatesRegularRows(), updatesStaticRow()); + addConditions(cbuilder.build(), request, options); + request.addRowUpdate(cbuilder, this, options, now); return request; } - public void addConditions(Composite clusteringPrefix, CQL3CasRequest request, QueryOptions options) throws InvalidRequestException + public void addConditions(Clustering clustering, CQL3CasRequest request, QueryOptions options) throws InvalidRequestException { if (ifNotExists) { // If we use ifNotExists, if the statement applies to any non static columns, then the condition is on the row of the non-static - // columns and the prefix should be the clusteringPrefix. But if only static columns are set, then the ifNotExists apply to the existence + // columns and the prefix should be the clustering. But if only static columns are set, then the ifNotExists apply to the existence // of any static columns and we should use the prefix for the "static part" of the partition. - request.addNotExist(clusteringPrefix); + request.addNotExist(clustering); } else if (ifExists) { - request.addExist(clusteringPrefix); + request.addExist(clustering); } else { if (columnConditions != null) - request.addConditions(clusteringPrefix, columnConditions, options); + request.addConditions(clustering, columnConditions, options); if (staticConditions != null) - request.addConditions(cfm.comparator.staticPrefix(), staticConditions, options); + request.addConditions(Clustering.STATIC_CLUSTERING, staticConditions, options); } } - private ResultSet buildCasResultSet(ByteBuffer key, ColumnFamily cf, QueryOptions options) throws InvalidRequestException + private ResultSet buildCasResultSet(RowIterator partition, QueryOptions options) throws InvalidRequestException { - return buildCasResultSet(keyspace(), key, columnFamily(), cf, getColumnsWithConditions(), false, options); + return buildCasResultSet(keyspace(), columnFamily(), partition, getColumnsWithConditions(), false, options); } - public static ResultSet buildCasResultSet(String ksName, ByteBuffer key, String cfName, ColumnFamily cf, Iterable columnsWithConditions, boolean isBatch, QueryOptions options) + public static ResultSet buildCasResultSet(String ksName, String tableName, RowIterator partition, Iterable columnsWithConditions, boolean isBatch, QueryOptions options) throws InvalidRequestException { - boolean success = cf == null; + boolean success = partition == null; - ColumnSpecification spec = new ColumnSpecification(ksName, cfName, CAS_RESULT_COLUMN, BooleanType.instance); + ColumnSpecification spec = new ColumnSpecification(ksName, tableName, CAS_RESULT_COLUMN, BooleanType.instance); ResultSet.ResultMetadata metadata = new ResultSet.ResultMetadata(Collections.singletonList(spec)); List> rows = Collections.singletonList(Collections.singletonList(BooleanType.instance.decompose(success))); ResultSet rs = new ResultSet(metadata, rows); - return success ? rs : merge(rs, buildCasFailureResultSet(key, cf, columnsWithConditions, isBatch, options)); + return success ? rs : merge(rs, buildCasFailureResultSet(partition, columnsWithConditions, isBatch, options)); } private static ResultSet merge(ResultSet left, ResultSet right) @@ -582,10 +688,10 @@ public abstract class ModificationStatement implements CQLStatement return new ResultSet(new ResultSet.ResultMetadata(specs), rows); } - private static ResultSet buildCasFailureResultSet(ByteBuffer key, ColumnFamily cf, Iterable columnsWithConditions, boolean isBatch, QueryOptions options) + private static ResultSet buildCasFailureResultSet(RowIterator partition, Iterable columnsWithConditions, boolean isBatch, QueryOptions options) throws InvalidRequestException { - CFMetaData cfm = cf.metadata(); + CFMetaData cfm = partition.metadata(); Selection selection; if (columnsWithConditions == null) { @@ -609,9 +715,8 @@ public abstract class ModificationStatement implements CQLStatement } - long now = System.currentTimeMillis(); - Selection.ResultSetBuilder builder = selection.resultSetBuilder(now, false); - SelectStatement.forSelection(cfm, selection).processColumnFamily(key, cf, options, now, builder); + Selection.ResultSetBuilder builder = selection.resultSetBuilder(false); + SelectStatement.forSelection(cfm, selection).processPartition(partition, options, builder, FBUtilities.nowInSeconds()); return builder.build(options.getProtocolVersion()); } @@ -640,31 +745,31 @@ public abstract class ModificationStatement implements CQLStatement public ResultMessage executeInternalWithCondition(QueryState state, QueryOptions options) throws RequestValidationException, RequestExecutionException { CQL3CasRequest request = makeCasRequest(state, options); - ColumnFamily result = casInternal(request, state); - return new ResultMessage.Rows(buildCasResultSet(request.key, result, options)); + try (RowIterator result = casInternal(request, state)) + { + return new ResultMessage.Rows(buildCasResultSet(result, options)); + } } - static ColumnFamily casInternal(CQL3CasRequest request, QueryState state) + static RowIterator casInternal(CQL3CasRequest request, QueryState state) { UUID ballot = UUIDGen.getTimeUUIDFromMicros(state.getTimestamp()); CFMetaData metadata = Schema.instance.getCFMetaData(request.cfm.ksName, request.cfm.cfName); - ReadCommand readCommand = ReadCommand.create(request.cfm.ksName, request.key, request.cfm.cfName, request.now, request.readFilter()); - Keyspace keyspace = Keyspace.open(request.cfm.ksName); - - Row row = readCommand.getRow(keyspace); - ColumnFamily current = row.cf; - if (!request.appliesTo(current)) + SinglePartitionReadCommand readCommand = request.readCommand(FBUtilities.nowInSeconds()); + FilteredPartition current; + try (ReadOrderGroup orderGroup = readCommand.startOrderGroup(); PartitionIterator iter = readCommand.executeInternal(orderGroup)) { - if (current == null) - current = ArrayBackedSortedColumns.factory.create(metadata); - return current; + current = FilteredPartition.create(PartitionIterators.getOnlyElement(iter, readCommand)); } - ColumnFamily updates = request.makeUpdates(current); - updates = TriggerExecutor.instance.execute(request.key, updates); + if (!request.appliesTo(current)) + return current.rowIterator(); - Commit proposal = Commit.newProposal(request.key, ballot, updates); + PartitionUpdate updates = request.makeUpdates(current); + updates = TriggerExecutor.instance.execute(updates); + + Commit proposal = Commit.newProposal(ballot, updates); proposal.makeMutation().apply(); return null; } @@ -683,17 +788,18 @@ public abstract class ModificationStatement implements CQLStatement throws RequestExecutionException, RequestValidationException { List keys = buildPartitionKeyNames(options); - Composite clusteringPrefix = createClusteringPrefix(options); + CBuilder clustering = createClustering(options); - UpdateParameters params = makeUpdateParameters(keys, clusteringPrefix, options, local, now); + UpdateParameters params = makeUpdateParameters(keys, clustering, options, local, now); Collection mutations = new ArrayList(keys.size()); for (ByteBuffer key: keys) { ThriftValidation.validateKey(cfm, key); - ColumnFamily cf = ArrayBackedSortedColumns.factory.create(cfm); - addUpdateForKey(cf, key, clusteringPrefix, params); - Mutation mut = new Mutation(cfm.ksName, key, cf); + DecoratedKey dk = StorageService.getPartitioner().decorateKey(key); + PartitionUpdate upd = new PartitionUpdate(cfm, dk, updatedColumns(), 1); + addUpdateForKey(upd, clustering, params); + Mutation mut = new Mutation(upd); mutations.add(isCounter() ? new CounterMutation(mut, options.getConsistency()) : mut); } @@ -701,15 +807,15 @@ public abstract class ModificationStatement implements CQLStatement } public UpdateParameters makeUpdateParameters(Collection keys, - Composite prefix, + CBuilder clustering, QueryOptions options, boolean local, long now) throws RequestExecutionException, RequestValidationException { // Some lists operation requires reading - Map rows = readRequiredRows(keys, prefix, local, options.getConsistency()); - return new UpdateParameters(cfm, options, getTimestamp(now, options), getTimeToLive(options), rows); + Map lists = readRequiredLists(keys, clustering, local, options.getConsistency()); + return new UpdateParameters(cfm, options, getTimestamp(now, options), getTimeToLive(options), lists, true); } /** @@ -803,6 +909,8 @@ public abstract class ModificationStatement implements CQLStatement stmt.validateWhereClauseForConditions(); } + + stmt.finishPreparation(); return stmt; } diff --git a/src/java/org/apache/cassandra/cql3/statements/SelectStatement.java b/src/java/org/apache/cassandra/cql3/statements/SelectStatement.java index e2708cd3f4..6a7f429500 100644 --- a/src/java/org/apache/cassandra/cql3/statements/SelectStatement.java +++ b/src/java/org/apache/cassandra/cql3/statements/SelectStatement.java @@ -23,7 +23,8 @@ import java.util.*; import com.google.common.base.Objects; import com.google.common.base.Predicate; import com.google.common.collect.Iterables; -import com.google.common.collect.Iterators; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; import org.apache.cassandra.auth.Permission; import org.apache.cassandra.config.CFMetaData; @@ -34,7 +35,8 @@ import org.apache.cassandra.cql3.restrictions.StatementRestrictions; import org.apache.cassandra.cql3.selection.RawSelector; import org.apache.cassandra.cql3.selection.Selection; import org.apache.cassandra.db.*; -import org.apache.cassandra.db.composites.*; +import org.apache.cassandra.db.rows.*; +import org.apache.cassandra.db.partitions.*; import org.apache.cassandra.db.filter.*; import org.apache.cassandra.db.index.SecondaryIndexManager; import org.apache.cassandra.db.marshal.CollectionType; @@ -43,12 +45,9 @@ import org.apache.cassandra.db.marshal.Int32Type; import org.apache.cassandra.dht.AbstractBounds; import org.apache.cassandra.exceptions.*; import org.apache.cassandra.serializers.MarshalException; -import org.apache.cassandra.service.ClientState; -import org.apache.cassandra.service.QueryState; -import org.apache.cassandra.service.StorageProxy; -import org.apache.cassandra.service.pager.Pageable; +import org.apache.cassandra.service.*; import org.apache.cassandra.service.pager.QueryPager; -import org.apache.cassandra.service.pager.QueryPagers; +import org.apache.cassandra.service.pager.PagingState; import org.apache.cassandra.thrift.ThriftValidation; import org.apache.cassandra.transport.messages.ResultMessage; import org.apache.cassandra.utils.ByteBufferUtil; @@ -71,6 +70,8 @@ import static org.apache.cassandra.utils.ByteBufferUtil.UNSET_BYTE_BUFFER; */ public class SelectStatement implements CQLStatement { + private static final Logger logger = LoggerFactory.getLogger(SelectStatement.class); + private static final int DEFAULT_COUNT_PAGE_SIZE = 10000; private final int boundTerms; @@ -88,6 +89,8 @@ public class SelectStatement implements CQLStatement */ private final Comparator> orderingComparator; + private final ColumnFilter queriedColumns; + // Used by forSelection below private static final Parameters defaultParameters = new Parameters(Collections.emptyMap(), false, false, false); @@ -108,6 +111,7 @@ public class SelectStatement implements CQLStatement this.orderingComparator = orderingComparator; this.parameters = parameters; this.limit = limit; + this.queriedColumns = gatherQueriedColumns(); } public Iterable getFunctions() @@ -117,6 +121,23 @@ public class SelectStatement implements CQLStatement limit != null ? limit.getFunctions() : Collections.emptySet()); } + // Note that the queried columns internally is different from the one selected by the + // user as it also include any column for which we have a restriction on. + private ColumnFilter gatherQueriedColumns() + { + if (selection.isWildcard()) + return ColumnFilter.all(cfm); + + ColumnFilter.Builder builder = ColumnFilter.allColumnsBuilder(cfm); + // Adds all selected columns + for (ColumnDefinition def : selection.getColumns()) + if (!def.isPrimaryKeyColumn()) + builder.add(def); + // as well as any restricted column (so we can actually apply the restriction) + builder.addAll(restrictions.nonPKRestrictedColumns()); + return builder.build(); + } + // Creates a simple select based on the given selection. // Note that the results select statement should not be used for actual queries, but only for processing already // queried data through processColumnFamily. @@ -161,31 +182,16 @@ public class SelectStatement implements CQLStatement cl.validateForRead(keyspace()); - int limit = getLimit(options); - long now = System.currentTimeMillis(); - Pageable command = getPageableCommand(options, limit, now); + int nowInSec = FBUtilities.nowInSeconds(); + ReadQuery query = getQuery(options, nowInSec); + int pageSize = getPageSize(options); - if (pageSize <= 0 || command == null || !QueryPagers.mayNeedPaging(command, pageSize)) - return execute(command, options, limit, now, state); + if (pageSize <= 0 || query.limits().count() <= pageSize) + return execute(query, options, state, nowInSec); - QueryPager pager = QueryPagers.pager(command, cl, state.getClientState(), options.getPagingState()); - return execute(pager, options, limit, now, pageSize); - } - - private Pageable getPageableCommand(QueryOptions options, int limit, long now) throws RequestValidationException - { - int limitForQuery = updateLimitForQuery(limit); - if (restrictions.isKeyRange() || restrictions.usesSecondaryIndexing()) - return getRangeCommand(options, limitForQuery, now); - - List commands = getSliceCommands(options, limitForQuery, now); - return commands == null ? null : new Pageable.ReadCommands(commands, limitForQuery); - } - - public Pageable getPageableCommand(QueryOptions options) throws RequestValidationException - { - return getPageableCommand(options, getLimit(options), System.currentTimeMillis()); + QueryPager pager = query.getPager(options.getPagingState()); + return execute(Pager.forDistributedQuery(pager, cl, state.getClientState()), options, pageSize, nowInSec); } private int getPageSize(QueryOptions options) @@ -201,103 +207,162 @@ public class SelectStatement implements CQLStatement return pageSize; } - private ResultMessage.Rows execute(Pageable command, QueryOptions options, int limit, long now, QueryState state) - throws RequestValidationException, RequestExecutionException + public ReadQuery getQuery(QueryOptions options, int nowInSec) throws RequestValidationException { - List rows; - if (command == null) - { - rows = Collections.emptyList(); - } - else - { - rows = command instanceof Pageable.ReadCommands - ? StorageProxy.read(((Pageable.ReadCommands)command).commands, options.getConsistency(), state.getClientState()) - : StorageProxy.getRangeSlice((RangeSliceCommand)command, options.getConsistency()); - } + DataLimits limit = getLimit(options); + if (restrictions.isKeyRange() || restrictions.usesSecondaryIndexing()) + return getRangeCommand(options, limit, nowInSec); - return processResults(rows, options, limit, now); + return getSliceCommands(options, limit, nowInSec); } - private ResultMessage.Rows execute(QueryPager pager, QueryOptions options, int limit, long now, int pageSize) + private ResultMessage.Rows execute(ReadQuery query, QueryOptions options, QueryState state, int nowInSec) throws RequestValidationException, RequestExecutionException + { + try (PartitionIterator data = query.execute(options.getConsistency(), state.getClientState())) + { + return processResults(data, options, nowInSec); + } + } + + // Simple wrapper class to avoid some code duplication + private static abstract class Pager + { + protected QueryPager pager; + + protected Pager(QueryPager pager) + { + this.pager = pager; + } + + public static Pager forInternalQuery(QueryPager pager, ReadOrderGroup orderGroup) + { + return new InternalPager(pager, orderGroup); + } + + public static Pager forDistributedQuery(QueryPager pager, ConsistencyLevel consistency, ClientState clientState) + { + return new NormalPager(pager, consistency, clientState); + } + + public boolean isExhausted() + { + return pager.isExhausted(); + } + + public PagingState state() + { + return pager.state(); + } + + public abstract PartitionIterator fetchPage(int pageSize); + + public static class NormalPager extends Pager + { + private final ConsistencyLevel consistency; + private final ClientState clientState; + + private NormalPager(QueryPager pager, ConsistencyLevel consistency, ClientState clientState) + { + super(pager); + this.consistency = consistency; + this.clientState = clientState; + } + + public PartitionIterator fetchPage(int pageSize) + { + return pager.fetchPage(pageSize, consistency, clientState); + } + } + + public static class InternalPager extends Pager + { + private final ReadOrderGroup orderGroup; + + private InternalPager(QueryPager pager, ReadOrderGroup orderGroup) + { + super(pager); + this.orderGroup = orderGroup; + } + + public PartitionIterator fetchPage(int pageSize) + { + return pager.fetchPageInternal(pageSize, orderGroup); + } + } + } + + private ResultMessage.Rows execute(Pager pager, QueryOptions options, int pageSize, int nowInSec) throws RequestValidationException, RequestExecutionException { if (selection.isAggregate()) - return pageAggregateQuery(pager, options, pageSize, now); + return pageAggregateQuery(pager, options, pageSize, nowInSec); // We can't properly do post-query ordering if we page (see #6722) checkFalse(needsPostQueryOrdering(), - "Cannot page queries with both ORDER BY and a IN restriction on the partition key;" - + " you must either remove the ORDER BY or the IN and sort client side, or disable paging for this query"); + "Cannot page queries with both ORDER BY and a IN restriction on the partition key;" + + " you must either remove the ORDER BY or the IN and sort client side, or disable paging for this query"); - List page = pager.fetchPage(pageSize); - ResultMessage.Rows msg = processResults(page, options, limit, now); + ResultMessage.Rows msg; + try (PartitionIterator page = pager.fetchPage(pageSize)) + { + msg = processResults(page, options, nowInSec); + } + // Please note that the isExhausted state of the pager only gets updated when we've closed the page, so this + // shouldn't be moved inside the 'try' above. if (!pager.isExhausted()) msg.result.metadata.setHasMorePages(pager.state()); return msg; } - private ResultMessage.Rows pageAggregateQuery(QueryPager pager, QueryOptions options, int pageSize, long now) - throws RequestValidationException, RequestExecutionException + private ResultMessage.Rows pageAggregateQuery(Pager pager, QueryOptions options, int pageSize, int nowInSec) + throws RequestValidationException, RequestExecutionException { - Selection.ResultSetBuilder result = selection.resultSetBuilder(now, parameters.isJson); + Selection.ResultSetBuilder result = selection.resultSetBuilder(parameters.isJson); while (!pager.isExhausted()) { - for (org.apache.cassandra.db.Row row : pager.fetchPage(pageSize)) + try (PartitionIterator iter = pager.fetchPage(pageSize)) { - // Not columns match the query, skip - if (row.cf == null) - continue; - - processColumnFamily(row.key.getKey(), row.cf, options, now, result); + while (iter.hasNext()) + processPartition(iter.next(), options, result, nowInSec); } } return new ResultMessage.Rows(result.build(options.getProtocolVersion())); } - public ResultMessage.Rows processResults(List rows, QueryOptions options, int limit, long now) throws RequestValidationException + private ResultMessage.Rows processResults(PartitionIterator partitions, QueryOptions options, int nowInSec) throws RequestValidationException { - ResultSet rset = process(rows, options, limit, now); + ResultSet rset = process(partitions, options, nowInSec); return new ResultMessage.Rows(rset); } - static List readLocally(String keyspaceName, List cmds) - { - Keyspace keyspace = Keyspace.open(keyspaceName); - List rows = new ArrayList(cmds.size()); - for (ReadCommand cmd : cmds) - rows.add(cmd.getRow(keyspace)); - return rows; - } - public ResultMessage.Rows executeInternal(QueryState state, QueryOptions options) throws RequestExecutionException, RequestValidationException { - int limit = getLimit(options); - long now = System.currentTimeMillis(); - Pageable command = getPageableCommand(options, limit, now); + int nowInSec = FBUtilities.nowInSeconds(); + ReadQuery query = getQuery(options, nowInSec); int pageSize = getPageSize(options); - if (pageSize <= 0 || command == null || !QueryPagers.mayNeedPaging(command, pageSize)) + try (ReadOrderGroup orderGroup = query.startOrderGroup()) { - List rows = command == null - ? Collections.emptyList() - : (command instanceof Pageable.ReadCommands - ? readLocally(keyspace(), ((Pageable.ReadCommands)command).commands) - : ((RangeSliceCommand)command).executeLocally()); - - return processResults(rows, options, limit, now); + if (pageSize <= 0 || query.limits().count() <= pageSize) + { + try (PartitionIterator data = query.executeInternal(orderGroup)) + { + return processResults(data, options, nowInSec); + } + } + else + { + QueryPager pager = query.getPager(options.getPagingState()); + return execute(Pager.forInternalQuery(pager, orderGroup), options, pageSize, nowInSec); + } } - - QueryPager pager = QueryPagers.localPager(command); - return execute(pager, options, limit, now, pageSize); } - public ResultSet process(List rows) throws InvalidRequestException + public ResultSet process(PartitionIterator partitions, int nowInSec) throws InvalidRequestException { - QueryOptions options = QueryOptions.DEFAULT; - return process(rows, options, getLimit(options), System.currentTimeMillis()); + return process(partitions, QueryOptions.DEFAULT, nowInSec); } public String keyspace() @@ -326,372 +391,239 @@ public class SelectStatement implements CQLStatement return restrictions; } - private List getSliceCommands(QueryOptions options, int limit, long now) throws RequestValidationException + private ReadQuery getSliceCommands(QueryOptions options, DataLimits limit, int nowInSec) throws RequestValidationException { Collection keys = restrictions.getPartitionKeys(options); + if (keys.isEmpty()) + return ReadQuery.EMPTY; - List commands = new ArrayList<>(keys.size()); - - IDiskAtomFilter filter = makeFilter(options, limit); + ClusteringIndexFilter filter = makeClusteringIndexFilter(options); if (filter == null) - return null; + return ReadQuery.EMPTY; + + RowFilter rowFilter = getRowFilter(options); // Note that we use the total limit for every key, which is potentially inefficient. // However, IN + LIMIT is not a very sensible choice. + List> commands = new ArrayList<>(keys.size()); for (ByteBuffer key : keys) { QueryProcessor.validateKey(key); - // We should not share the slice filter amongst the commands (hence the cloneShallow), due to - // SliceQueryFilter not being immutable due to its columnCounter used by the lastCounted() method - // (this is fairly ugly and we should change that but that's probably not a tiny refactor to do that cleanly) - commands.add(ReadCommand.create(keyspace(), ByteBufferUtil.clone(key), columnFamily(), now, filter.cloneShallow())); + DecoratedKey dk = StorageService.getPartitioner().decorateKey(ByteBufferUtil.clone(key)); + commands.add(SinglePartitionReadCommand.create(cfm, nowInSec, queriedColumns, rowFilter, limit, dk, filter)); } - return commands; + return new SinglePartitionReadCommand.Group(commands, limit); } - private RangeSliceCommand getRangeCommand(QueryOptions options, int limit, long now) throws RequestValidationException + private ReadQuery getRangeCommand(QueryOptions options, DataLimits limit, int nowInSec) throws RequestValidationException { - IDiskAtomFilter filter = makeFilter(options, limit); - if (filter == null) - return null; + ClusteringIndexFilter clusteringIndexFilter = makeClusteringIndexFilter(options); + if (clusteringIndexFilter == null) + return ReadQuery.EMPTY; - List expressions = getValidatedIndexExpressions(options); + RowFilter rowFilter = getRowFilter(options); // The LIMIT provided by the user is the number of CQL row he wants returned. // We want to have getRangeSlice to count the number of columns, not the number of keys. - AbstractBounds keyBounds = restrictions.getPartitionKeyBounds(options); + AbstractBounds keyBounds = restrictions.getPartitionKeyBounds(options); return keyBounds == null - ? null - : new RangeSliceCommand(keyspace(), columnFamily(), now, filter, keyBounds, expressions, limit, !parameters.isDistinct, false); + ? ReadQuery.EMPTY + : new PartitionRangeReadCommand(cfm, nowInSec, queriedColumns, rowFilter, limit, new DataRange(keyBounds, clusteringIndexFilter)); } - private ColumnSlice makeStaticSlice() - { - // Note: we could use staticPrefix.start() for the start bound, but EMPTY gives us the - // same effect while saving a few CPU cycles. - return isReversed - ? new ColumnSlice(cfm.comparator.staticPrefix().end(), Composites.EMPTY) - : new ColumnSlice(Composites.EMPTY, cfm.comparator.staticPrefix().end()); - } - - private IDiskAtomFilter makeFilter(QueryOptions options, int limit) + private ClusteringIndexFilter makeClusteringIndexFilter(QueryOptions options) throws InvalidRequestException { - int toGroup = cfm.comparator.isDense() ? -1 : cfm.clusteringColumns().size(); if (parameters.isDistinct) { - // For distinct, we only care about fetching the beginning of each partition. If we don't have - // static columns, we in fact only care about the first cell, so we query only that (we don't "group"). - // If we do have static columns, we do need to fetch the first full group (to have the static columns values). - - // See the comments on IGNORE_TOMBSTONED_PARTITIONS and CASSANDRA-8490 for why we use a special value for - // DISTINCT queries on the partition key only. - toGroup = selection.containsStaticColumns() ? toGroup : SliceQueryFilter.IGNORE_TOMBSTONED_PARTITIONS; - return new SliceQueryFilter(ColumnSlice.ALL_COLUMNS_ARRAY, false, 1, toGroup); + // We need to be able to distinguish between partition having live rows and those that don't. But + // doing so is not trivial since "having a live row" depends potentially on + // 1) when the query is performed, due to TTLs + // 2) how thing reconcile together between different nodes + // so that it's hard to really optimize properly internally. So to keep it simple, we simply query + // for the first row of the partition and hence uses Slices.ALL. We'll limit it to the first live + // row however in getLimit(). + return new ClusteringIndexSliceFilter(Slices.ALL, false); } - else if (restrictions.isColumnRange()) + + if (restrictions.isColumnRange()) { - List startBounds = restrictions.getClusteringColumnsBoundsAsComposites(Bound.START, options); - List endBounds = restrictions.getClusteringColumnsBoundsAsComposites(Bound.END, options); - assert startBounds.size() == endBounds.size(); + Slices slices = makeSlices(options); + if (slices == Slices.NONE && !selection.containsStaticColumns()) + return null; - // Handles fetching static columns. Note that for 2i, the filter is just used to restrict - // the part of the index to query so adding the static slice would be useless and confusing. - // For 2i, static columns are retrieve in CompositesSearcher with each index hit. - ColumnSlice staticSlice = selection.containsStaticColumns() && !restrictions.usesSecondaryIndexing() - ? makeStaticSlice() - : null; - - // The case where startBounds == 1 is common enough that it's worth optimizing - if (startBounds.size() == 1) - { - ColumnSlice slice = new ColumnSlice(startBounds.get(0), endBounds.get(0)); - if (slice.isAlwaysEmpty(cfm.comparator, isReversed)) - return staticSlice == null ? null : sliceFilter(staticSlice, limit, toGroup); - - if (staticSlice == null) - return sliceFilter(slice, limit, toGroup); - - if (isReversed) - return slice.includes(cfm.comparator.reverseComparator(), staticSlice.start) - ? sliceFilter(new ColumnSlice(slice.start, staticSlice.finish), limit, toGroup) - : sliceFilter(new ColumnSlice[]{ slice, staticSlice }, limit, toGroup); - else - return slice.includes(cfm.comparator, staticSlice.finish) - ? sliceFilter(new ColumnSlice(staticSlice.start, slice.finish), limit, toGroup) - : sliceFilter(new ColumnSlice[]{ staticSlice, slice }, limit, toGroup); - } - - List l = new ArrayList(startBounds.size()); - for (int i = 0; i < startBounds.size(); i++) - { - ColumnSlice slice = new ColumnSlice(startBounds.get(i), endBounds.get(i)); - if (!slice.isAlwaysEmpty(cfm.comparator, isReversed)) - l.add(slice); - } - - if (l.isEmpty()) - return staticSlice == null ? null : sliceFilter(staticSlice, limit, toGroup); - if (staticSlice == null) - return sliceFilter(l.toArray(new ColumnSlice[l.size()]), limit, toGroup); - - // The slices should not overlap. We know the slices built from startBounds/endBounds don't, but if there is - // a static slice, it could overlap with the 2nd slice. Check for it and correct if that's the case - ColumnSlice[] slices; - if (isReversed) - { - if (l.get(l.size() - 1).includes(cfm.comparator.reverseComparator(), staticSlice.start)) - { - slices = l.toArray(new ColumnSlice[l.size()]); - slices[slices.length-1] = new ColumnSlice(slices[slices.length-1].start, Composites.EMPTY); - } - else - { - slices = l.toArray(new ColumnSlice[l.size()+1]); - slices[slices.length-1] = staticSlice; - } - } - else - { - if (l.get(0).includes(cfm.comparator, staticSlice.finish)) - { - slices = new ColumnSlice[l.size()]; - slices[0] = new ColumnSlice(Composites.EMPTY, l.get(0).finish); - for (int i = 1; i < l.size(); i++) - slices[i] = l.get(i); - } - else - { - slices = new ColumnSlice[l.size()+1]; - slices[0] = staticSlice; - for (int i = 0; i < l.size(); i++) - slices[i+1] = l.get(i); - } - } - return sliceFilter(slices, limit, toGroup); + return new ClusteringIndexSliceFilter(slices, isReversed); } else { - SortedSet cellNames = getRequestedColumns(options); - if (cellNames == null) // in case of IN () for the last column of the key + NavigableSet clusterings = getRequestedRows(options); + if (clusterings.isEmpty() && !selection.containsStaticColumns()) // in case of IN () for the last column of the key return null; - QueryProcessor.validateCellNames(cellNames, cfm.comparator); - return new NamesQueryFilter(cellNames, true); + + return new ClusteringIndexNamesFilter(clusterings, isReversed); } } - private SliceQueryFilter sliceFilter(ColumnSlice slice, int limit, int toGroup) + private Slices makeSlices(QueryOptions options) + throws InvalidRequestException { - return sliceFilter(new ColumnSlice[]{ slice }, limit, toGroup); - } + SortedSet startBounds = restrictions.getClusteringColumnsBounds(Bound.START, options); + SortedSet endBounds = restrictions.getClusteringColumnsBounds(Bound.END, options); + assert startBounds.size() == endBounds.size(); - private SliceQueryFilter sliceFilter(ColumnSlice[] slices, int limit, int toGroup) - { - assert ColumnSlice.validateSlices(slices, cfm.comparator, isReversed) : String.format("Invalid slices: " + Arrays.toString(slices) + (isReversed ? " (reversed)" : "")); - return new SliceQueryFilter(slices, isReversed, limit, toGroup); + // The case where startBounds == 1 is common enough that it's worth optimizing + if (startBounds.size() == 1) + { + Slice.Bound start = startBounds.first(); + Slice.Bound end = endBounds.first(); + return cfm.comparator.compare(start, end) > 0 + ? Slices.NONE + : Slices.with(cfm.comparator, Slice.make(start, end)); + } + + Slices.Builder builder = new Slices.Builder(cfm.comparator, startBounds.size()); + Iterator startIter = startBounds.iterator(); + Iterator endIter = endBounds.iterator(); + while (startIter.hasNext() && endIter.hasNext()) + { + Slice.Bound start = startIter.next(); + Slice.Bound end = endIter.next(); + + // Ignore slices that are nonsensical + if (cfm.comparator.compare(start, end) > 0) + continue; + + builder.add(start, end); + } + + return builder.build(); } /** * May be used by custom QueryHandler implementations */ - public int getLimit(QueryOptions options) throws InvalidRequestException + public DataLimits getLimit(QueryOptions options) throws InvalidRequestException { - if (limit != null) + int userLimit = -1; + // If we aggregate, the limit really apply to the number of rows returned to the user, not to what is queried, and + // since in practice we currently only aggregate at top level (we have no GROUP BY support yet), we'll only ever + // return 1 result and can therefore basically ignore the user LIMIT in this case. + // Whenever we support GROUP BY, we'll have to add a new DataLimits kind that knows how things are grouped and is thus + // able to apply the user limit properly. + if (limit != null && !selection.isAggregate()) { ByteBuffer b = checkNotNull(limit.bindAndGet(options), "Invalid null value of limit"); // treat UNSET limit value as 'unlimited' - if (b == UNSET_BYTE_BUFFER) - return Integer.MAX_VALUE; - try + if (b != UNSET_BYTE_BUFFER) { - Int32Type.instance.validate(b); - int l = Int32Type.instance.compose(b); - checkTrue(l > 0, "LIMIT must be strictly positive"); - return l; - } - catch (MarshalException e) - { - throw new InvalidRequestException("Invalid limit value"); + try + { + Int32Type.instance.validate(b); + userLimit = Int32Type.instance.compose(b); + checkTrue(userLimit > 0, "LIMIT must be strictly positive"); + } + catch (MarshalException e) + { + throw new InvalidRequestException("Invalid limit value"); + } } } - return Integer.MAX_VALUE; + + if (parameters.isDistinct) + return userLimit < 0 ? DataLimits.DISTINCT_NONE : DataLimits.distinctLimits(userLimit); + + return userLimit < 0 ? DataLimits.NONE : DataLimits.cqlLimits(userLimit); } - private int updateLimitForQuery(int limit) - { - // Internally, we don't support exclusive bounds for slices. Instead, we query one more element if necessary - // and exclude it later (in processColumnFamily) - return restrictions.isNonCompositeSliceWithExclusiveBounds() && limit != Integer.MAX_VALUE - ? limit + 1 - : limit; - } - - private SortedSet getRequestedColumns(QueryOptions options) throws InvalidRequestException + private NavigableSet getRequestedRows(QueryOptions options) throws InvalidRequestException { // Note: getRequestedColumns don't handle static columns, but due to CASSANDRA-5762 // we always do a slice for CQL3 tables, so it's ok to ignore them here assert !restrictions.isColumnRange(); - SortedSet columns = new TreeSet(cfm.comparator); - for (Composite composite : restrictions.getClusteringColumnsAsComposites(options)) - columns.addAll(addSelectedColumns(composite)); - return columns; - } - - private SortedSet addSelectedColumns(Composite prefix) - { - if (cfm.comparator.isDense()) - { - return FBUtilities.singleton(cfm.comparator.create(prefix, null), cfm.comparator); - } - else - { - SortedSet columns = new TreeSet(cfm.comparator); - - // We need to query the selected column as well as the marker - // column (for the case where the row exists but has no columns outside the PK) - // Two exceptions are "static CF" (non-composite non-compact CF) and "super CF" - // that don't have marker and for which we must query all columns instead - if (cfm.comparator.isCompound() && !cfm.isSuper()) - { - // marker - columns.add(cfm.comparator.rowMarker(prefix)); - - // selected columns - for (ColumnDefinition def : selection.getColumns()) - if (def.isRegular() || def.isStatic()) - columns.add(cfm.comparator.create(prefix, def)); - } - else - { - // We now that we're not composite so we can ignore static columns - for (ColumnDefinition def : cfm.regularColumns()) - columns.add(cfm.comparator.create(prefix, def)); - } - return columns; - } + return restrictions.getClusteringColumns(options); } /** * May be used by custom QueryHandler implementations */ - public List getValidatedIndexExpressions(QueryOptions options) throws InvalidRequestException + public RowFilter getRowFilter(QueryOptions options) throws InvalidRequestException { - if (!restrictions.usesSecondaryIndexing()) - return Collections.emptyList(); - ColumnFamilyStore cfs = Keyspace.open(keyspace()).getColumnFamilyStore(columnFamily()); SecondaryIndexManager secondaryIndexManager = cfs.indexManager; - - List expressions = restrictions.getIndexExpressions(secondaryIndexManager, options); - - secondaryIndexManager.validateIndexSearchersForQuery(expressions); - - return expressions; + RowFilter filter = restrictions.getRowFilter(secondaryIndexManager, options); + secondaryIndexManager.validateFilter(filter); + return filter; } - private CellName makeExclusiveSliceBound(Bound bound, CellNameType type, QueryOptions options) throws InvalidRequestException + private ResultSet process(PartitionIterator partitions, QueryOptions options, int nowInSec) throws InvalidRequestException { - if (restrictions.areRequestedBoundsInclusive(bound)) - return null; - - return type.makeCellName(restrictions.getClusteringColumnsBounds(bound, options).get(0)); - } - - private Iterator applySliceRestriction(final Iterator cells, final QueryOptions options) throws InvalidRequestException - { - final CellNameType type = cfm.comparator; - - final CellName excludedStart = makeExclusiveSliceBound(Bound.START, type, options); - final CellName excludedEnd = makeExclusiveSliceBound(Bound.END, type, options); - - return Iterators.filter(cells, new Predicate() + Selection.ResultSetBuilder result = selection.resultSetBuilder(parameters.isJson); + while (partitions.hasNext()) { - public boolean apply(Cell c) + try (RowIterator partition = partitions.next()) { - // For dynamic CF, the column could be out of the requested bounds (because we don't support strict bounds internally (unless - // the comparator is composite that is)), filter here - return !((excludedStart != null && type.compare(c.name(), excludedStart) == 0) - || (excludedEnd != null && type.compare(c.name(), excludedEnd) == 0)); + processPartition(partition, options, result, nowInSec); } - }); - } - - private ResultSet process(List rows, QueryOptions options, int limit, long now) throws InvalidRequestException - { - Selection.ResultSetBuilder result = selection.resultSetBuilder(now, parameters.isJson); - for (org.apache.cassandra.db.Row row : rows) - { - // Not columns match the query, skip - if (row.cf == null) - continue; - - processColumnFamily(row.key.getKey(), row.cf, options, now, result); } ResultSet cqlRows = result.build(options.getProtocolVersion()); orderResults(cqlRows); - // Internal calls always return columns in the comparator order, even when reverse was set - if (isReversed) - cqlRows.reverse(); - - // Trim result if needed to respect the user limit - cqlRows.trim(limit); return cqlRows; } - // Used by ModificationStatement for CAS operations - void processColumnFamily(ByteBuffer key, ColumnFamily cf, QueryOptions options, long now, Selection.ResultSetBuilder result) - throws InvalidRequestException + public static ByteBuffer[] getComponents(CFMetaData cfm, DecoratedKey dk) { - CFMetaData cfm = cf.metadata(); - ByteBuffer[] keyComponents = null; + ByteBuffer key = dk.getKey(); if (cfm.getKeyValidator() instanceof CompositeType) { - keyComponents = ((CompositeType)cfm.getKeyValidator()).split(key); + return ((CompositeType)cfm.getKeyValidator()).split(key); } else { - keyComponents = new ByteBuffer[]{ key }; + return new ByteBuffer[]{ key }; } + } - Iterator cells = cf.getSortedColumns().iterator(); - if (restrictions.isNonCompositeSliceWithExclusiveBounds()) - cells = applySliceRestriction(cells, options); - + // Used by ModificationStatement for CAS operations + void processPartition(RowIterator partition, QueryOptions options, Selection.ResultSetBuilder result, int nowInSec) + throws InvalidRequestException + { int protocolVersion = options.getProtocolVersion(); - CQL3Row.RowIterator iter = cfm.comparator.CQL3RowBuilder(cfm, now).group(cells); - // If there is static columns but there is no non-static row, then provided the select was a full - // partition selection (i.e. not a 2ndary index search and there was no condition on clustering columns) - // then we want to include the static columns in the result set (and we're done). - CQL3Row staticRow = iter.getStaticRow(); - if (staticRow != null && !iter.hasNext() && !restrictions.usesSecondaryIndexing() && restrictions.hasNoClusteringColumnsRestriction()) + ByteBuffer[] keyComponents = getComponents(cfm, partition.partitionKey()); + + Row staticRow = partition.staticRow().takeAlias(); + // If there is no rows, then provided the select was a full partition selection + // (i.e. not a 2ndary index search and there was no condition on clustering columns), + // we want to include static columns and we're done. + if (!partition.hasNext()) { - result.newRow(protocolVersion); - for (ColumnDefinition def : selection.getColumns()) + if (!staticRow.isEmpty() && (!restrictions.usesSecondaryIndexing() || cfm.isStaticCompactTable()) && !restrictions.hasClusteringColumnsRestriction()) { - switch (def.kind) + result.newRow(protocolVersion); + for (ColumnDefinition def : selection.getColumns()) { - case PARTITION_KEY: - result.add(keyComponents[def.position()]); - break; - case STATIC: - addValue(result, def, staticRow, options); - break; - default: - result.add((ByteBuffer)null); + switch (def.kind) + { + case PARTITION_KEY: + result.add(keyComponents[def.position()]); + break; + case STATIC: + addValue(result, def, staticRow, nowInSec, protocolVersion); + break; + default: + result.add((ByteBuffer)null); + } } } return; } - while (iter.hasNext()) + while (partition.hasNext()) { - CQL3Row cql3Row = iter.next(); - - // Respect requested order + Row row = partition.next(); result.newRow(protocolVersion); // Respect selection order for (ColumnDefinition def : selection.getColumns()) @@ -702,41 +634,35 @@ public class SelectStatement implements CQLStatement result.add(keyComponents[def.position()]); break; case CLUSTERING_COLUMN: - result.add(cql3Row.getClusteringColumn(def.position())); - break; - case COMPACT_VALUE: - result.add(cql3Row.getColumn(null)); + result.add(row.clustering().get(def.position())); break; case REGULAR: - addValue(result, def, cql3Row, options); + addValue(result, def, row, nowInSec, protocolVersion); break; case STATIC: - addValue(result, def, staticRow, options); + addValue(result, def, staticRow, nowInSec, protocolVersion); break; } } } } - private static void addValue(Selection.ResultSetBuilder result, ColumnDefinition def, CQL3Row row, QueryOptions options) + private static void addValue(Selection.ResultSetBuilder result, ColumnDefinition def, Row row, int nowInSec, int protocolVersion) { - if (row == null) + if (def.isComplex()) { - result.add((ByteBuffer)null); - return; + // Collections are the only complex types we have so far + assert def.type.isCollection() && def.type.isMultiCell(); + Iterator cells = row.getCells(def); + if (cells == null) + result.add((ByteBuffer)null); + else + result.add(((CollectionType)def.type).serializeForNativeProtocol(def, cells, protocolVersion)); } - - if (def.type.isMultiCell()) + else { - List cells = row.getMultiCellColumn(def.name); - ByteBuffer buffer = cells == null - ? null - : ((CollectionType)def.type).serializeForNativeProtocol(def, cells, options.getProtocolVersion()); - result.add(buffer); - return; + result.add(row.getCell(def), nowInSec); } - - result.add(row.getColumn(def.name)); } private boolean needsPostQueryOrdering() @@ -796,9 +722,6 @@ public class SelectStatement implements CQLStatement isReversed = isReversed(cfm); } - if (isReversed) - restrictions.reverse(); - checkNeedsFiltering(restrictions); SelectStatement stmt = new SelectStatement(cfm, @@ -832,7 +755,8 @@ public class SelectStatement implements CQLStatement whereClause, boundNames, selection.containsOnlyStaticColumns(), - selection.containsACollection()); + selection.containsACollection(), + parameters.allowFiltering); } catch (UnrecognizedEntityException e) { @@ -980,47 +904,12 @@ public class SelectStatement implements CQLStatement { // We will potentially filter data if either: // - Have more than one IndexExpression - // - Have no index expression and the column filter is not the identity + // - Have no index expression and the row filter is not the identity checkFalse(restrictions.needFiltering(), "Cannot execute this query as it might involve data filtering and " + "thus may have unpredictable performance. If you want to execute " + "this query despite the performance unpredictability, use ALLOW FILTERING"); } - - // We don't internally support exclusive slice bounds on non-composite tables. To deal with it we do an - // inclusive slice and remove post-query the value that shouldn't be returned. One problem however is that - // if there is a user limit, that limit may make the query return before the end of the slice is reached, - // in which case, once we'll have removed bound post-query, we might end up with less results than - // requested which would be incorrect. For single-partition query, this is not a problem, we just ask for - // one more result (see updateLimitForQuery()) since that's enough to compensate for that problem. For key - // range however, each returned row may include one result that will have to be trimmed, so we would have - // to bump the query limit by N where N is the number of rows we will return, but we don't know that in - // advance. So, since we currently don't have a good way to handle such query, we refuse it (#7059) rather - // than answering with something that is wrong. - if (restrictions.isNonCompositeSliceWithExclusiveBounds() && restrictions.isKeyRange() && limit != null) - { - SingleColumnRelation rel = findInclusiveClusteringRelationForCompact(restrictions.cfm); - throw invalidRequest("The query requests a restriction of rows with a strict bound (%s) over a range of partitions. " - + "This is not supported by the underlying storage engine for COMPACT tables if a LIMIT is provided. " - + "Please either make the condition non strict (%s) or remove the user LIMIT", rel, rel.withNonStrictOperator()); - } - } - - private SingleColumnRelation findInclusiveClusteringRelationForCompact(CFMetaData cfm) - { - for (Relation r : whereClause) - { - // We only call this when sliceRestriction != null, i.e. for compact table with non composite comparator, - // so it can't be a MultiColumnRelation. - SingleColumnRelation rel = (SingleColumnRelation)r; - - if (cfm.getColumnDefinition(rel.getEntity().prepare(cfm)).isClusteringColumn() - && (rel.operator() == Operator.GT || rel.operator() == Operator.LT)) - return rel; - } - - // We're not supposed to call this method unless we know this can't happen - throw new AssertionError(); } private boolean containsAlias(final ColumnIdentifier name) diff --git a/src/java/org/apache/cassandra/cql3/statements/UpdateStatement.java b/src/java/org/apache/cassandra/cql3/statements/UpdateStatement.java index ad46a0f0eb..bbaacc873e 100644 --- a/src/java/org/apache/cassandra/cql3/statements/UpdateStatement.java +++ b/src/java/org/apache/cassandra/cql3/statements/UpdateStatement.java @@ -17,22 +17,18 @@ */ package org.apache.cassandra.cql3.statements; -import java.nio.ByteBuffer; import java.util.*; import org.apache.cassandra.cql3.*; import org.apache.cassandra.config.CFMetaData; import org.apache.cassandra.config.ColumnDefinition; import org.apache.cassandra.db.*; -import org.apache.cassandra.db.composites.Composite; -import org.apache.cassandra.db.index.SecondaryIndex; -import org.apache.cassandra.db.index.SecondaryIndexManager; +import org.apache.cassandra.db.rows.*; +import org.apache.cassandra.db.partitions.PartitionUpdate; import org.apache.cassandra.exceptions.*; import org.apache.cassandra.utils.ByteBufferUtil; import org.apache.cassandra.utils.Pair; -import static org.apache.cassandra.cql3.statements.RequestValidations.invalidRequest; - /** * An UPDATE statement parsed from a CQL query statement. * @@ -51,98 +47,52 @@ public class UpdateStatement extends ModificationStatement return true; } - public void addUpdateForKey(ColumnFamily cf, - ByteBuffer key, - Composite prefix, - UpdateParameters params) throws InvalidRequestException + public void addUpdateForKey(PartitionUpdate update, CBuilder cbuilder, UpdateParameters params) + throws InvalidRequestException { - addUpdateForKey(cf, key, prefix, params, true); - } + params.newPartition(update.partitionKey()); - public void addUpdateForKey(ColumnFamily cf, - ByteBuffer key, - Composite prefix, - UpdateParameters params, - boolean validateIndexedColumns) throws InvalidRequestException - { - // Inserting the CQL row marker (see #4361) - // We always need to insert a marker for INSERT, because of the following situation: - // CREATE TABLE t ( k int PRIMARY KEY, c text ); - // INSERT INTO t(k, c) VALUES (1, 1) - // DELETE c FROM t WHERE k = 1; - // SELECT * FROM t; - // The last query should return one row (but with c == null). Adding the marker with the insert make sure - // the semantic is correct (while making sure a 'DELETE FROM t WHERE k = 1' does remove the row entirely) - // - // We do not insert the marker for UPDATE however, as this amount to updating the columns in the WHERE - // clause which is inintuitive (#6782) - // - // We never insert markers for Super CF as this would confuse the thrift side. - if (type == StatementType.INSERT && cfm.isCQL3Table() && !prefix.isStatic()) - cf.addColumn(params.makeColumn(cfm.comparator.rowMarker(prefix), ByteBufferUtil.EMPTY_BYTE_BUFFER)); - - List updates = getOperations(); - - if (cfm.comparator.isDense()) + if (updatesRegularRows()) { - if (prefix.isEmpty()) - throw new InvalidRequestException(String.format("Missing PRIMARY KEY part %s", cfm.clusteringColumns().get(0))); + Clustering clustering = cbuilder.build(); + Row.Writer writer = update.writer(); + params.writeClustering(clustering, writer); - // An empty name for the compact value is what we use to recognize the case where there is not column - // outside the PK, see CreateStatement. - if (!cfm.compactValueColumn().name.bytes.hasRemaining()) + + // We update the row timestamp (ex-row marker) only on INSERT (#6782) + // Further, COMPACT tables semantic differs from "CQL3" ones in that a row exists only if it has + // a non-null column, so we don't want to set the row timestamp for them. + if (type == StatementType.INSERT && cfm.isCQLTable()) + params.writePartitionKeyLivenessInfo(writer); + + List updates = getRegularOperations(); + + // For compact tablw, when we translate it to thrift, we don't have a row marker. So we don't accept an insert/update + // that only sets the PK unless the is no declared non-PK columns (in the latter we just set the value empty). + + // For a dense layout, when we translate it to thrift, we don't have a row marker. So we don't accept an insert/update + // that only sets the PK unless the is no declared non-PK columns (which we recognize because in that case the compact + // value is of type "EmptyType"). + if (cfm.isCompactTable() && updates.isEmpty()) { - // There is no column outside the PK. So no operation could have passed through validation - assert updates.isEmpty(); - new Constants.Setter(cfm.compactValueColumn(), EMPTY).execute(key, cf, prefix, params); - } - else - { - // dense means we don't have a row marker, so don't accept to set only the PK. See CASSANDRA-5648. - if (updates.isEmpty()) + if (CompactTables.hasEmptyCompactValue(cfm)) + updates = Collections.singletonList(new Constants.Setter(cfm.compactValueColumn(), EMPTY)); + else throw new InvalidRequestException(String.format("Column %s is mandatory for this COMPACT STORAGE table", cfm.compactValueColumn().name)); - - for (Operation update : updates) - update.execute(key, cf, prefix, params); } - } - else - { - for (Operation update : updates) - update.execute(key, cf, prefix, params); + + for (Operation op : updates) + op.execute(update.partitionKey(), clustering, writer, params); + + writer.endOfRow(); } - // validateIndexedColumns trigger a call to Keyspace.open() which we want to be able to avoid in some case - //(e.g. when using CQLSSTableWriter) - if (validateIndexedColumns) - validateIndexedColumns(key, cf); - } - - /** - * Checks if the values of the indexed columns are valid. - * - * @param key row key for the column family - * @param cf the column family - * @throws InvalidRequestException if one of the values of the indexed columns is not valid - */ - private void validateIndexedColumns(ByteBuffer key, ColumnFamily cf) - { - SecondaryIndexManager indexManager = Keyspace.open(cfm.ksName).getColumnFamilyStore(cfm.cfId).indexManager; - if (indexManager.hasIndexes()) + if (updatesStaticRow()) { - for (Cell cell : cf) - { - // Indexed values must be validated by any applicable index. See CASSANDRA-3057/4240/8081 for more details - SecondaryIndex failedIndex = indexManager.validate(key, cell); - if (failedIndex != null) - { - throw invalidRequest(String.format("Can't index column value of size %d for index %s on %s.%s", - cell.value().remaining(), - failedIndex.getIndexName(), - cfm.ksName, - cfm.cfName)); - } - } + Row.Writer writer = update.staticWriter(); + for (Operation op : getStaticOperations()) + op.execute(update.partitionKey(), Clustering.STATIC_CLUSTERING, writer, params); + writer.endOfRow(); } } diff --git a/src/java/org/apache/cassandra/db/AbstractCell.java b/src/java/org/apache/cassandra/db/AbstractCell.java deleted file mode 100644 index bd63985174..0000000000 --- a/src/java/org/apache/cassandra/db/AbstractCell.java +++ /dev/null @@ -1,236 +0,0 @@ -/** - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.apache.cassandra.db; - -import java.io.DataInput; -import java.io.IOError; -import java.io.IOException; -import java.nio.ByteBuffer; -import java.security.MessageDigest; -import java.util.Iterator; - -import com.google.common.collect.AbstractIterator; -import org.apache.cassandra.config.CFMetaData; -import org.apache.cassandra.db.composites.CellName; -import org.apache.cassandra.db.composites.CellNameType; -import org.apache.cassandra.db.context.CounterContext; -import org.apache.cassandra.db.marshal.AbstractType; -import org.apache.cassandra.io.sstable.format.Version; -import org.apache.cassandra.serializers.MarshalException; -import org.apache.cassandra.utils.FBUtilities; - -public abstract class AbstractCell implements Cell -{ - public static Iterator onDiskIterator(final DataInput in, - final ColumnSerializer.Flag flag, - final int expireBefore, - final Version version, - final CellNameType type) - { - return new AbstractIterator() - { - protected OnDiskAtom computeNext() - { - OnDiskAtom atom; - try - { - atom = type.onDiskAtomSerializer().deserializeFromSSTable(in, flag, expireBefore, version); - } - catch (IOException e) - { - throw new IOError(e); - } - if (atom == null) - return endOfData(); - - return atom; - } - }; - } - - public boolean isLive() - { - return true; - } - - public boolean isLive(long now) - { - return true; - } - - public int cellDataSize() - { - return name().dataSize() + value().remaining() + TypeSizes.NATIVE.sizeof(timestamp()); - } - - public int serializedSize(CellNameType type, TypeSizes typeSizes) - { - /* - * Size of a column is = - * size of a name (short + length of the string) - * + 1 byte to indicate if the column has been deleted - * + 8 bytes for timestamp - * + 4 bytes which basically indicates the size of the byte array - * + entire byte array. - */ - int valueSize = value().remaining(); - return ((int)type.cellSerializer().serializedSize(name(), typeSizes)) + 1 + typeSizes.sizeof(timestamp()) + typeSizes.sizeof(valueSize) + valueSize; - } - - public int serializationFlags() - { - return 0; - } - - public Cell diff(Cell cell) - { - if (timestamp() < cell.timestamp()) - return cell; - return null; - } - - public void updateDigest(MessageDigest digest) - { - digest.update(name().toByteBuffer().duplicate()); - digest.update(value().duplicate()); - - FBUtilities.updateWithLong(digest, timestamp()); - FBUtilities.updateWithByte(digest, serializationFlags()); - } - - public int getLocalDeletionTime() - { - return Integer.MAX_VALUE; - } - - public Cell reconcile(Cell cell) - { - long ts1 = timestamp(), ts2 = cell.timestamp(); - if (ts1 != ts2) - return ts1 < ts2 ? cell : this; - if (isLive() != cell.isLive()) - return isLive() ? cell : this; - return value().compareTo(cell.value()) < 0 ? cell : this; - } - - @Override - public boolean equals(Object o) - { - return this == o || (o instanceof Cell && equals((Cell) o)); - } - - public boolean equals(Cell cell) - { - return timestamp() == cell.timestamp() && name().equals(cell.name()) && value().equals(cell.value()) - && serializationFlags() == cell.serializationFlags(); - } - - public int hashCode() - { - throw new UnsupportedOperationException(); - } - - public String getString(CellNameType comparator) - { - return String.format("%s:%b:%d@%d", - comparator.getString(name()), - !isLive(), - value().remaining(), - timestamp()); - } - - public void validateName(CFMetaData metadata) throws MarshalException - { - metadata.comparator.validate(name()); - } - - public void validateFields(CFMetaData metadata) throws MarshalException - { - validateName(metadata); - - AbstractType valueValidator = metadata.getValueValidator(name()); - if (valueValidator != null) - valueValidator.validateCellValue(value()); - } - - public static Cell create(CellName name, ByteBuffer value, long timestamp, int ttl, CFMetaData metadata) - { - if (ttl <= 0) - ttl = metadata.getDefaultTimeToLive(); - - return ttl > 0 - ? new BufferExpiringCell(name, value, timestamp, ttl) - : new BufferCell(name, value, timestamp); - } - - public Cell diffCounter(Cell cell) - { - assert this instanceof CounterCell : "Wrong class type: " + getClass(); - - if (timestamp() < cell.timestamp()) - return cell; - - // Note that if at that point, cell can't be a tombstone. Indeed, - // cell is the result of merging us with other nodes results, and - // merging a CounterCell with a tombstone never return a tombstone - // unless that tombstone timestamp is greater that the CounterCell - // one. - assert cell instanceof CounterCell : "Wrong class type: " + cell.getClass(); - - if (((CounterCell) this).timestampOfLastDelete() < ((CounterCell) cell).timestampOfLastDelete()) - return cell; - - CounterContext.Relationship rel = CounterCell.contextManager.diff(cell.value(), value()); - return (rel == CounterContext.Relationship.GREATER_THAN || rel == CounterContext.Relationship.DISJOINT) ? cell : null; - } - - /** This is temporary until we start creating Cells of the different type (buffer vs. native) */ - public Cell reconcileCounter(Cell cell) - { - assert this instanceof CounterCell : "Wrong class type: " + getClass(); - - // No matter what the counter cell's timestamp is, a tombstone always takes precedence. See CASSANDRA-7346. - if (cell instanceof DeletedCell) - return cell; - - assert (cell instanceof CounterCell) : "Wrong class type: " + cell.getClass(); - - // live < live last delete - if (timestamp() < ((CounterCell) cell).timestampOfLastDelete()) - return cell; - - long timestampOfLastDelete = ((CounterCell) this).timestampOfLastDelete(); - - // live last delete > live - if (timestampOfLastDelete > cell.timestamp()) - return this; - - // live + live. return one of the cells if its context is a superset of the other's, or merge them otherwise - ByteBuffer context = CounterCell.contextManager.merge(value(), cell.value()); - if (context == value() && timestamp() >= cell.timestamp() && timestampOfLastDelete >= ((CounterCell) cell).timestampOfLastDelete()) - return this; - else if (context == cell.value() && cell.timestamp() >= timestamp() && ((CounterCell) cell).timestampOfLastDelete() >= timestampOfLastDelete) - return cell; - else // merge clocks and timestamps. - return new BufferCounterCell(name(), - context, - Math.max(timestamp(), cell.timestamp()), - Math.max(timestampOfLastDelete, ((CounterCell) cell).timestampOfLastDelete())); - } - -} diff --git a/src/java/org/apache/cassandra/db/AbstractClusteringPrefix.java b/src/java/org/apache/cassandra/db/AbstractClusteringPrefix.java new file mode 100644 index 0000000000..9ea071df7d --- /dev/null +++ b/src/java/org/apache/cassandra/db/AbstractClusteringPrefix.java @@ -0,0 +1,93 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.cassandra.db; + +import java.nio.ByteBuffer; +import java.security.MessageDigest; +import java.util.Objects; + +import org.apache.cassandra.utils.FBUtilities; + +public abstract class AbstractClusteringPrefix implements ClusteringPrefix +{ + public ClusteringPrefix clustering() + { + return this; + } + + public int dataSize() + { + int size = 0; + for (int i = 0; i < size(); i++) + { + ByteBuffer bb = get(i); + size += bb == null ? 0 : bb.remaining(); + } + return size; + } + + public void digest(MessageDigest digest) + { + for (int i = 0; i < size(); i++) + { + ByteBuffer bb = get(i); + if (bb != null) + digest.update(bb.duplicate()); + } + FBUtilities.updateWithByte(digest, kind().ordinal()); + } + + public void writeTo(Writer writer) + { + for (int i = 0; i < size(); i++) + writer.writeClusteringValue(get(i)); + } + + public long unsharedHeapSize() + { + // unsharedHeapSize is used inside the cache and in memtables. Implementations that are + // safe to use there (SimpleClustering, Slice.Bound.SimpleBound and MemtableRow.* classes) overwrite this. + throw new UnsupportedOperationException(); + } + + @Override + public final int hashCode() + { + int result = 31; + for (int i = 0; i < size(); i++) + result += 31 * Objects.hashCode(get(i)); + return 31 * result + Objects.hashCode(kind()); + } + + @Override + public final boolean equals(Object o) + { + if(!(o instanceof ClusteringPrefix)) + return false; + + ClusteringPrefix that = (ClusteringPrefix)o; + if (this.kind() != that.kind() || this.size() != that.size()) + return false; + + for (int i = 0; i < size(); i++) + if (!Objects.equals(this.get(i), that.get(i))) + return false; + + return true; + } +} diff --git a/src/java/org/apache/cassandra/db/AbstractLivenessInfo.java b/src/java/org/apache/cassandra/db/AbstractLivenessInfo.java new file mode 100644 index 0000000000..bbda598555 --- /dev/null +++ b/src/java/org/apache/cassandra/db/AbstractLivenessInfo.java @@ -0,0 +1,164 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.cassandra.db; + +import java.util.Objects; +import java.security.MessageDigest; + +import org.apache.cassandra.serializers.MarshalException; +import org.apache.cassandra.utils.FBUtilities; + +/** + * Base abstract class for {@code LivenessInfo} implementations. + * + * All {@code LivenessInfo} should extends this class unless it has a very + * good reason not to. + */ +public abstract class AbstractLivenessInfo implements LivenessInfo +{ + public boolean hasTimestamp() + { + return timestamp() != NO_TIMESTAMP; + } + + public boolean hasTTL() + { + return ttl() != NO_TTL; + } + + public boolean hasLocalDeletionTime() + { + return localDeletionTime() != NO_DELETION_TIME; + } + + public int remainingTTL(int nowInSec) + { + if (!hasTTL()) + return -1; + + int remaining = localDeletionTime() - nowInSec; + return remaining >= 0 ? remaining : -1; + } + + public boolean isLive(int nowInSec) + { + // Note that we don't rely on localDeletionTime() only because if we were to, we + // could potentially consider a tombstone as a live cell (due to time skew). So + // if a cell has a local deletion time and no ttl, it's a tombstone and consider + // dead no matter what it's actual local deletion value is. + return hasTimestamp() && (!hasLocalDeletionTime() || (hasTTL() && nowInSec < localDeletionTime())); + } + + public void digest(MessageDigest digest) + { + FBUtilities.updateWithLong(digest, timestamp()); + FBUtilities.updateWithInt(digest, localDeletionTime()); + FBUtilities.updateWithInt(digest, ttl()); + } + + public void validate() + { + if (ttl() < 0) + throw new MarshalException("A TTL should not be negative"); + if (localDeletionTime() < 0) + throw new MarshalException("A local deletion time should not be negative"); + if (hasTTL() && !hasLocalDeletionTime()) + throw new MarshalException("Shoud not have a TTL without an associated local deletion time"); + } + + public int dataSize() + { + int size = 0; + if (hasTimestamp()) + size += TypeSizes.NATIVE.sizeof(timestamp()); + if (hasTTL()) + size += TypeSizes.NATIVE.sizeof(ttl()); + if (hasLocalDeletionTime()) + size += TypeSizes.NATIVE.sizeof(localDeletionTime()); + return size; + + } + + public boolean supersedes(LivenessInfo other) + { + return timestamp() > other.timestamp(); + } + + public LivenessInfo mergeWith(LivenessInfo other) + { + return supersedes(other) ? this : other; + } + + public LivenessInfo takeAlias() + { + return new SimpleLivenessInfo(timestamp(), ttl(), localDeletionTime()); + }; + + public LivenessInfo withUpdatedTimestamp(long newTimestamp) + { + if (!hasTimestamp()) + return this; + + return new SimpleLivenessInfo(newTimestamp, ttl(), localDeletionTime()); + } + + public boolean isPurgeable(long maxPurgeableTimestamp, int gcBefore) + { + return timestamp() < maxPurgeableTimestamp && localDeletionTime() < gcBefore; + } + + @Override + public String toString() + { + StringBuilder sb = new StringBuilder(); + sb.append('['); + boolean needSpace = false; + if (hasTimestamp()) + { + sb.append("ts=").append(timestamp()); + needSpace = true; + } + if (hasTTL()) + { + sb.append(needSpace ? ' ' : "").append("ttl=").append(ttl()); + needSpace = true; + } + if (hasLocalDeletionTime()) + sb.append(needSpace ? ' ' : "").append("ldt=").append(localDeletionTime()); + sb.append(']'); + return sb.toString(); + } + + @Override + public boolean equals(Object other) + { + if(!(other instanceof LivenessInfo)) + return false; + + LivenessInfo that = (LivenessInfo)other; + return this.timestamp() == that.timestamp() + && this.ttl() == that.ttl() + && this.localDeletionTime() == that.localDeletionTime(); + } + + @Override + public int hashCode() + { + return Objects.hash(timestamp(), ttl(), localDeletionTime()); + } +} diff --git a/src/java/org/apache/cassandra/db/AbstractNativeCell.java b/src/java/org/apache/cassandra/db/AbstractNativeCell.java deleted file mode 100644 index 207a972b4c..0000000000 --- a/src/java/org/apache/cassandra/db/AbstractNativeCell.java +++ /dev/null @@ -1,710 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.apache.cassandra.db; - -import java.nio.ByteBuffer; -import java.nio.ByteOrder; -import java.security.MessageDigest; - -import net.nicoulaj.compilecommand.annotations.Inline; -import org.apache.cassandra.config.CFMetaData; -import org.apache.cassandra.config.ColumnDefinition; -import org.apache.cassandra.cql3.ColumnIdentifier; -import org.apache.cassandra.db.composites.*; -import org.apache.cassandra.db.filter.ColumnSlice; -import org.apache.cassandra.db.marshal.AbstractType; -import org.apache.cassandra.db.marshal.CompositeType; -import org.apache.cassandra.utils.ByteBufferUtil; -import org.apache.cassandra.utils.FBUtilities; -import org.apache.cassandra.utils.FastByteOperations; -import org.apache.cassandra.utils.concurrent.OpOrder; -import org.apache.cassandra.utils.memory.*; - - -/** - * 
- * {@code
- * Packs a CellName AND a Cell into one off-heap representation.
- * Layout is:
- *
- * Note we store the ColumnIdentifier in full as bytes. This seems an okay tradeoff for now, as we just
- * look it back up again when we need to, and in the near future we hope to switch to ints, longs or
- * UUIDs representing column identifiers on disk, at which point we can switch that here as well.
- *
- * [timestamp][value offset][name size]][name extra][name offset deltas][cell names][value][Descendants]
- * [   8b    ][     4b     ][    2b   ][     1b    ][     each 2b      ][ arb < 64k][ arb ][ arbitrary ]
- *
- * descendants: any overriding classes will put their state here
- * name offsets are deltas from their base offset, and don't include the first offset, or the end position of the final entry,
- * i.e. there will be size - 1 entries, and each is a delta that is added to the offset of the position of the first name
- * (which is always CELL_NAME_OFFSETS_OFFSET + (2 * (size - 1))). The length of the final name fills up any remaining
- * space upto the value offset
- * name extra:  lowest 2 bits indicate the clustering size delta (i.e. how many name items are NOT part of the clustering key)
- *              the next 2 bits indicate the CellNameType
- *              the next bit indicates if the column is a static or clustered/dynamic column
- * }
- *
- */ -public abstract class AbstractNativeCell extends AbstractCell implements CellName -{ - static final int TIMESTAMP_OFFSET = 4; - private static final int VALUE_OFFSET_OFFSET = 12; - private static final int CELL_NAME_SIZE_OFFSET = 16; - private static final int CELL_NAME_EXTRA_OFFSET = 18; - private static final int CELL_NAME_OFFSETS_OFFSET = 19; - private static final int CELL_NAME_SIZE_DELTA_MASK = 3; - private static final int CELL_NAME_TYPE_SHIFT = 2; - private static final int CELL_NAME_TYPE_MASK = 7; - - private static enum NameType - { - COMPOUND_DENSE(0 << 2), COMPOUND_SPARSE(1 << 2), COMPOUND_SPARSE_STATIC(2 << 2), SIMPLE_DENSE(3 << 2), SIMPLE_SPARSE(4 << 2); - static final NameType[] TYPES = NameType.values(); - final int bits; - - NameType(int bits) - { - this.bits = bits; - } - - static NameType typeOf(CellName name) - { - if (name instanceof CompoundDenseCellName) - { - assert !name.isStatic(); - return COMPOUND_DENSE; - } - - if (name instanceof CompoundSparseCellName) - return name.isStatic() ? COMPOUND_SPARSE_STATIC : COMPOUND_SPARSE; - - if (name instanceof SimpleDenseCellName) - { - assert !name.isStatic(); - return SIMPLE_DENSE; - } - - if (name instanceof SimpleSparseCellName) - { - assert !name.isStatic(); - return SIMPLE_SPARSE; - } - - if (name instanceof NativeCell) - return ((NativeCell) name).nametype(); - - throw new AssertionError(); - } - } - - private final long peer; // peer is assigned by peer updater in setPeer method - - AbstractNativeCell() - { - peer = -1; - } - - public AbstractNativeCell(NativeAllocator allocator, OpOrder.Group writeOp, Cell copyOf) - { - int size = sizeOf(copyOf); - peer = allocator.allocate(size, writeOp); - - MemoryUtil.setInt(peer, size); - construct(copyOf); - } - - protected int sizeOf(Cell cell) - { - int size = CELL_NAME_OFFSETS_OFFSET + Math.max(0, cell.name().size() - 1) * 2 + cell.value().remaining(); - CellName name = cell.name(); - for (int i = 0; i < name.size(); i++) - size += name.get(i).remaining(); - return size; - } - - protected void construct(Cell from) - { - setLong(TIMESTAMP_OFFSET, from.timestamp()); - CellName name = from.name(); - int nameSize = name.size(); - int offset = CELL_NAME_SIZE_OFFSET; - setShort(offset, (short) nameSize); - assert nameSize - name.clusteringSize() <= 2; - byte cellNameExtraBits = (byte) ((nameSize - name.clusteringSize()) | NameType.typeOf(name).bits); - setByte(offset += 2, cellNameExtraBits); - offset += 1; - short cellNameDelta = 0; - for (int i = 1; i < nameSize; i++) - { - cellNameDelta += name.get(i - 1).remaining(); - setShort(offset, cellNameDelta); - offset += 2; - } - for (int i = 0; i < nameSize; i++) - { - ByteBuffer bb = name.get(i); - setBytes(offset, bb); - offset += bb.remaining(); - } - setInt(VALUE_OFFSET_OFFSET, offset); - setBytes(offset, from.value()); - } - - // the offset at which to read the short that gives the names - private int nameDeltaOffset(int i) - { - return CELL_NAME_OFFSETS_OFFSET + ((i - 1) * 2); - } - - int valueStartOffset() - { - return getInt(VALUE_OFFSET_OFFSET); - } - - private int valueEndOffset() - { - return (int) (internalSize() - postfixSize()); - } - - protected int postfixSize() - { - return 0; - } - - @Override - public ByteBuffer value() - { - long offset = valueStartOffset(); - return getByteBuffer(offset, (int) (internalSize() - (postfixSize() + offset))).order(ByteOrder.BIG_ENDIAN); - } - - private int clusteringSizeDelta() - { - return getByte(CELL_NAME_EXTRA_OFFSET) & CELL_NAME_SIZE_DELTA_MASK; - } - - public boolean isStatic() - { - return nametype() == NameType.COMPOUND_SPARSE_STATIC; - } - - NameType nametype() - { - return NameType.TYPES[(((int) this.getByte(CELL_NAME_EXTRA_OFFSET)) >> CELL_NAME_TYPE_SHIFT) & CELL_NAME_TYPE_MASK]; - } - - public long minTimestamp() - { - return timestamp(); - } - - public long maxTimestamp() - { - return timestamp(); - } - - public int clusteringSize() - { - return size() - clusteringSizeDelta(); - } - - @Override - public ColumnIdentifier cql3ColumnName(CFMetaData metadata) - { - switch (nametype()) - { - case SIMPLE_SPARSE: - return getIdentifier(metadata, get(clusteringSize())); - case COMPOUND_SPARSE_STATIC: - case COMPOUND_SPARSE: - ByteBuffer buffer = get(clusteringSize()); - if (buffer.remaining() == 0) - return CompoundSparseCellNameType.rowMarkerId; - - return getIdentifier(metadata, buffer); - case SIMPLE_DENSE: - case COMPOUND_DENSE: - return null; - default: - throw new AssertionError(); - } - } - - public ByteBuffer collectionElement() - { - return isCollectionCell() ? get(size() - 1) : null; - } - - // we always have a collection element if our clustering size is 2 less than our total size, - // and we never have one otherwiss - public boolean isCollectionCell() - { - return clusteringSizeDelta() == 2; - } - - public boolean isSameCQL3RowAs(CellNameType type, CellName other) - { - switch (nametype()) - { - case SIMPLE_DENSE: - case COMPOUND_DENSE: - return type.compare(this, other) == 0; - case COMPOUND_SPARSE_STATIC: - case COMPOUND_SPARSE: - int clusteringSize = clusteringSize(); - if (clusteringSize != other.clusteringSize() || other.isStatic() != isStatic()) - return false; - for (int i = 0; i < clusteringSize; i++) - if (type.subtype(i).compare(get(i), other.get(i)) != 0) - return false; - return true; - case SIMPLE_SPARSE: - return true; - default: - throw new AssertionError(); - } - } - - public int size() - { - return getShort(CELL_NAME_SIZE_OFFSET); - } - - public boolean isEmpty() - { - return size() == 0; - } - - public ByteBuffer get(int i) - { - return get(i, null); - } - - private ByteBuffer get(int i, AbstractAllocator copy) - { - // remember to take dense/sparse into account, and only return EOC when not dense - int size = size(); - assert i >= 0 && i < size(); - int cellNamesOffset = nameDeltaOffset(size); - int startDelta = i == 0 ? 0 : getShort(nameDeltaOffset(i)); - int endDelta = i < size - 1 ? getShort(nameDeltaOffset(i + 1)) : valueStartOffset() - cellNamesOffset; - int length = endDelta - startDelta; - if (copy == null) - return getByteBuffer(cellNamesOffset + startDelta, length).order(ByteOrder.BIG_ENDIAN); - ByteBuffer result = copy.allocate(length); - FastByteOperations.UnsafeOperations.copy(null, peer + cellNamesOffset + startDelta, result, 0, length); - return result; - } - - private static final ThreadLocal BUFFER = new ThreadLocal() - { - protected byte[] initialValue() - { - return new byte[256]; - } - }; - - protected void writeComponentTo(MessageDigest digest, int i, boolean includeSize) - { - // remember to take dense/sparse into account, and only return EOC when not dense - int size = size(); - assert i >= 0 && i < size(); - int cellNamesOffset = nameDeltaOffset(size); - int startDelta = i == 0 ? 0 : getShort(nameDeltaOffset(i)); - int endDelta = i < size - 1 ? getShort(nameDeltaOffset(i + 1)) : valueStartOffset() - cellNamesOffset; - - int componentStart = cellNamesOffset + startDelta; - int count = endDelta - startDelta; - - if (includeSize) - FBUtilities.updateWithShort(digest, count); - - writeMemoryTo(digest, componentStart, count); - } - - protected void writeMemoryTo(MessageDigest digest, int from, int count) - { - // only batch if we have more than 16 bytes remaining to transfer, otherwise fall-back to single-byte updates - int i = 0, batchEnd = count - 16; - if (i < batchEnd) - { - byte[] buffer = BUFFER.get(); - while (i < batchEnd) - { - int transfer = Math.min(count - i, 256); - getBytes(from + i, buffer, 0, transfer); - digest.update(buffer, 0, transfer); - i += transfer; - } - } - while (i < count) - digest.update(getByte(from + i++)); - } - - public EOC eoc() - { - return EOC.NONE; - } - - public Composite withEOC(EOC eoc) - { - throw new UnsupportedOperationException(); - } - - public Composite start() - { - throw new UnsupportedOperationException(); - } - - public Composite end() - { - throw new UnsupportedOperationException(); - } - - public ColumnSlice slice() - { - throw new UnsupportedOperationException(); - } - - public boolean isPrefixOf(CType type, Composite c) - { - if (size() > c.size() || isStatic() != c.isStatic()) - return false; - - for (int i = 0; i < size(); i++) - { - if (type.subtype(i).compare(get(i), c.get(i)) != 0) - return false; - } - return true; - } - - public ByteBuffer toByteBuffer() - { - // for simple sparse we just return our one name buffer - switch (nametype()) - { - case SIMPLE_DENSE: - case SIMPLE_SPARSE: - return get(0); - case COMPOUND_DENSE: - case COMPOUND_SPARSE_STATIC: - case COMPOUND_SPARSE: - // This is the legacy format of composites. - // See org.apache.cassandra.db.marshal.CompositeType for details. - ByteBuffer result = ByteBuffer.allocate(cellDataSize()); - if (isStatic()) - ByteBufferUtil.writeShortLength(result, CompositeType.STATIC_MARKER); - - for (int i = 0; i < size(); i++) - { - ByteBuffer bb = get(i); - ByteBufferUtil.writeShortLength(result, bb.remaining()); - result.put(bb); - result.put((byte) 0); - } - result.flip(); - return result; - default: - throw new AssertionError(); - } - } - - protected void updateWithName(MessageDigest digest) - { - // for simple sparse we just return our one name buffer - switch (nametype()) - { - case SIMPLE_DENSE: - case SIMPLE_SPARSE: - writeComponentTo(digest, 0, false); - break; - - case COMPOUND_DENSE: - case COMPOUND_SPARSE_STATIC: - case COMPOUND_SPARSE: - // This is the legacy format of composites. - // See org.apache.cassandra.db.marshal.CompositeType for details. - if (isStatic()) - FBUtilities.updateWithShort(digest, CompositeType.STATIC_MARKER); - - for (int i = 0; i < size(); i++) - { - writeComponentTo(digest, i, true); - digest.update((byte) 0); - } - break; - - default: - throw new AssertionError(); - } - } - - protected void updateWithValue(MessageDigest digest) - { - int offset = valueStartOffset(); - int length = valueEndOffset() - offset; - writeMemoryTo(digest, offset, length); - } - - @Override // this is the NAME dataSize, only! - public int dataSize() - { - switch (nametype()) - { - case SIMPLE_DENSE: - case SIMPLE_SPARSE: - return valueStartOffset() - nameDeltaOffset(size()); - case COMPOUND_DENSE: - case COMPOUND_SPARSE_STATIC: - case COMPOUND_SPARSE: - int size = size(); - return valueStartOffset() - nameDeltaOffset(size) + 3 * size + (isStatic() ? 2 : 0); - default: - throw new AssertionError(); - } - } - - public boolean equals(Object obj) - { - if (obj == this) - return true; - if (obj instanceof CellName) - return equals((CellName) obj); - if (obj instanceof Cell) - return equals((Cell) obj); - return false; - } - - public boolean equals(CellName that) - { - int size = this.size(); - if (size != that.size()) - return false; - - for (int i = 0 ; i < size ; i++) - if (!get(i).equals(that.get(i))) - return false; - return true; - } - - private static final ByteBuffer[] EMPTY = new ByteBuffer[0]; - - @Override - public CellName copy(CFMetaData cfm, AbstractAllocator allocator) - { - ByteBuffer[] r; - switch (nametype()) - { - case SIMPLE_DENSE: - return CellNames.simpleDense(get(0, allocator)); - - case COMPOUND_DENSE: - r = new ByteBuffer[size()]; - for (int i = 0; i < r.length; i++) - r[i] = get(i, allocator); - return CellNames.compositeDense(r); - - case COMPOUND_SPARSE_STATIC: - case COMPOUND_SPARSE: - int clusteringSize = clusteringSize(); - r = clusteringSize == 0 ? EMPTY : new ByteBuffer[clusteringSize()]; - for (int i = 0; i < clusteringSize; i++) - r[i] = get(i, allocator); - - ByteBuffer nameBuffer = get(r.length); - ColumnIdentifier name; - - if (nameBuffer.remaining() == 0) - { - name = CompoundSparseCellNameType.rowMarkerId; - } - else - { - name = getIdentifier(cfm, nameBuffer); - } - - if (clusteringSizeDelta() == 2) - { - ByteBuffer element = allocator.clone(get(size() - 1)); - return CellNames.compositeSparseWithCollection(r, element, name, isStatic()); - } - return CellNames.compositeSparse(r, name, isStatic()); - - case SIMPLE_SPARSE: - return CellNames.simpleSparse(getIdentifier(cfm, get(0))); - } - throw new IllegalStateException(); - } - - private static ColumnIdentifier getIdentifier(CFMetaData cfMetaData, ByteBuffer name) - { - ColumnDefinition def = cfMetaData.getColumnDefinition(name); - if (def != null) - { - return def.name; - } - else - { - // it's safe to simply grab based on clusteringPrefixSize() as we are only called if not a dense type - AbstractType type = cfMetaData.comparator.subtype(cfMetaData.comparator.clusteringPrefixSize()); - return new ColumnIdentifier(HeapAllocator.instance.clone(name), type); - } - } - - @Override - public Cell withUpdatedName(CellName newName) - { - throw new UnsupportedOperationException(); - } - - @Override - public Cell withUpdatedTimestamp(long newTimestamp) - { - throw new UnsupportedOperationException(); - } - - protected long internalSize() - { - return MemoryUtil.getInt(peer); - } - - private void checkPosition(long offset, long size) - { - assert size >= 0; - assert peer > 0 : "Memory was freed"; - assert offset >= 0 && offset + size <= internalSize() : String.format("Illegal range: [%d..%d), size: %s", offset, offset + size, internalSize()); - } - - protected final void setByte(long offset, byte b) - { - checkPosition(offset, 1); - MemoryUtil.setByte(peer + offset, b); - } - - protected final void setShort(long offset, short s) - { - checkPosition(offset, 1); - MemoryUtil.setShort(peer + offset, s); - } - - protected final void setInt(long offset, int l) - { - checkPosition(offset, 4); - MemoryUtil.setInt(peer + offset, l); - } - - protected final void setLong(long offset, long l) - { - checkPosition(offset, 8); - MemoryUtil.setLong(peer + offset, l); - } - - protected final void setBytes(long offset, ByteBuffer buffer) - { - int start = buffer.position(); - int count = buffer.limit() - start; - if (count == 0) - return; - - checkPosition(offset, count); - MemoryUtil.setBytes(peer + offset, buffer); - } - - protected final byte getByte(long offset) - { - checkPosition(offset, 1); - return MemoryUtil.getByte(peer + offset); - } - - protected final void getBytes(long offset, byte[] trg, int trgOffset, int count) - { - checkPosition(offset, count); - MemoryUtil.getBytes(peer + offset, trg, trgOffset, count); - } - - protected final int getShort(long offset) - { - checkPosition(offset, 2); - return MemoryUtil.getShort(peer + offset); - } - - protected final int getInt(long offset) - { - checkPosition(offset, 4); - return MemoryUtil.getInt(peer + offset); - } - - protected final long getLong(long offset) - { - checkPosition(offset, 8); - return MemoryUtil.getLong(peer + offset); - } - - protected final ByteBuffer getByteBuffer(long offset, int length) - { - checkPosition(offset, length); - return MemoryUtil.getByteBuffer(peer + offset, length); - } - - // requires isByteOrderComparable to be true. Compares the name components only; ; may need to compare EOC etc still - @Inline - public final int compareTo(final Composite that) - { - if (isStatic() != that.isStatic()) - { - // Static sorts before non-static no matter what, except for empty which - // always sort first - if (isEmpty()) - return that.isEmpty() ? 0 : -1; - if (that.isEmpty()) - return 1; - return isStatic() ? -1 : 1; - } - - int size = size(); - int size2 = that.size(); - int minSize = Math.min(size, size2); - int startDelta = 0; - int cellNamesOffset = nameDeltaOffset(size); - for (int i = 0 ; i < minSize ; i++) - { - int endDelta = i < size - 1 ? getShort(nameDeltaOffset(i + 1)) : valueStartOffset() - cellNamesOffset; - long offset = peer + cellNamesOffset + startDelta; - int length = endDelta - startDelta; - int cmp = FastByteOperations.UnsafeOperations.compareTo(null, offset, length, that.get(i)); - if (cmp != 0) - return cmp; - startDelta = endDelta; - } - - EOC eoc = that.eoc(); - if (size == size2) - return this.eoc().compareTo(eoc); - - return size < size2 ? this.eoc().prefixComparisonResult : -eoc.prefixComparisonResult; - } - - public final int compareToSimple(final Composite that) - { - assert size() == 1 && that.size() == 1; - int length = valueStartOffset() - nameDeltaOffset(1); - long offset = peer + nameDeltaOffset(1); - return FastByteOperations.UnsafeOperations.compareTo(null, offset, length, that.get(0)); - } -} diff --git a/src/java/org/apache/cassandra/db/AbstractRangeCommand.java b/src/java/org/apache/cassandra/db/AbstractRangeCommand.java deleted file mode 100644 index 8bcb5b3c10..0000000000 --- a/src/java/org/apache/cassandra/db/AbstractRangeCommand.java +++ /dev/null @@ -1,101 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.apache.cassandra.db; - -import java.util.List; - -import org.apache.cassandra.config.DatabaseDescriptor; -import org.apache.cassandra.db.filter.*; -import org.apache.cassandra.db.index.*; -import org.apache.cassandra.dht.AbstractBounds; -import org.apache.cassandra.net.MessageOut; -import org.apache.cassandra.service.IReadCommand; - -public abstract class AbstractRangeCommand implements IReadCommand -{ - public final String keyspace; - public final String columnFamily; - public final long timestamp; - - public final AbstractBounds keyRange; - public final IDiskAtomFilter predicate; - public final List rowFilter; - - public final SecondaryIndexSearcher searcher; - - public AbstractRangeCommand(String keyspace, String columnFamily, long timestamp, AbstractBounds keyRange, IDiskAtomFilter predicate, List rowFilter) - { - this.keyspace = keyspace; - this.columnFamily = columnFamily; - this.timestamp = timestamp; - this.keyRange = keyRange; - this.predicate = predicate; - this.rowFilter = rowFilter; - SecondaryIndexManager indexManager = Keyspace.open(keyspace).getColumnFamilyStore(columnFamily).indexManager; - this.searcher = indexManager.getHighestSelectivityIndexSearcher(rowFilter); - } - - public boolean requiresScanningAllRanges() - { - return searcher != null && searcher.requiresScanningAllRanges(rowFilter); - } - - public List postReconciliationProcessing(List rows) - { - return searcher == null ? trim(rows) : trim(searcher.postReconciliationProcessing(rowFilter, rows)); - } - - private List trim(List rows) - { - if (countCQL3Rows() || ignoredTombstonedPartitions()) - return rows; - else - return rows.size() > limit() ? rows.subList(0, limit()) : rows; - } - - public String getKeyspace() - { - return keyspace; - } - - public abstract MessageOut createMessage(); - public abstract AbstractRangeCommand forSubRange(AbstractBounds range); - public abstract AbstractRangeCommand withUpdatedLimit(int newLimit); - - public abstract int limit(); - public abstract boolean countCQL3Rows(); - - /** - * Returns true if tombstoned partitions should not be included in results or count towards the limit. - * See CASSANDRA-8490 for more details on why this is needed (and done this way). - * */ - public boolean ignoredTombstonedPartitions() - { - if (!(predicate instanceof SliceQueryFilter)) - return false; - - return ((SliceQueryFilter) predicate).compositesToGroup == SliceQueryFilter.IGNORE_TOMBSTONED_PARTITIONS; - } - - public abstract List executeLocally(); - - public long getTimeout() - { - return DatabaseDescriptor.getRangeRpcTimeout(); - } -} diff --git a/src/java/org/apache/cassandra/db/Aliasable.java b/src/java/org/apache/cassandra/db/Aliasable.java new file mode 100644 index 0000000000..a4396fc172 --- /dev/null +++ b/src/java/org/apache/cassandra/db/Aliasable.java @@ -0,0 +1,62 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.cassandra.db; + +/** + * This interface marks objects that are only valid in a restricted scope and + * shouldn't be simply aliased outside of this scope (in other words, you should + * not keep a reference to the object that escaped said scope as the object will + * likely become invalid). + * + * For instance, most {@link RowIterator} implementation reuse the same {@link + * Row} object during iteration. This means that the following code would be + * incorrect. + * 
+ *    RowIterator iter = ...;
+ *    Row someRow = null;
+ *    while (iter.hasNext())
+ *    {
+ *        Row row = iter.next();
+ *        if (someCondition(row))
+ *           someRow = row;         // This isn't safe
+ *        doSomethingElse();
+ *    }
+ *    useRow(someRow);
+ *
+ * The problem being that, because the row iterator reuse the same object, + * {@code someRow} will not point to the row that had met {@code someCondition} + * at the end of the iteration ({@code someRow} will point to the last iterated + * row in practice). + * + * When code do need to alias such {@code Aliasable} object, it should call the + * {@code takeAlias} method that will make a copy of the object if necessary. + * + * Of course, the {@code takeAlias} should not be abused, as it defeat the purpose + * of sharing objects in the first place. + * + * Also note that some implementation of an {@code Aliasable} object may be + * safe to alias, in which case its {@code takeAlias} method will be a no-op. + */ +public interface Aliasable +{ + /** + * Returns either this object (if it's safe to alias) or a copy of it + * (it it isn't safe to alias). + */ + public T takeAlias(); +} diff --git a/src/java/org/apache/cassandra/db/ArrayBackedSortedColumns.java b/src/java/org/apache/cassandra/db/ArrayBackedSortedColumns.java deleted file mode 100644 index 1beb982d46..0000000000 --- a/src/java/org/apache/cassandra/db/ArrayBackedSortedColumns.java +++ /dev/null @@ -1,774 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.apache.cassandra.db; - -import java.util.*; - -import com.google.common.base.Function; -import com.google.common.collect.AbstractIterator; -import com.google.common.collect.Iterables; - -import org.apache.cassandra.config.CFMetaData; -import org.apache.cassandra.db.composites.CellName; -import org.apache.cassandra.db.composites.Composite; -import org.apache.cassandra.db.filter.ColumnSlice; -import org.apache.cassandra.utils.BatchRemoveIterator; -import org.apache.cassandra.utils.memory.AbstractAllocator; -import org.apache.cassandra.utils.SearchIterator; - -/** - * A ColumnFamily backed by an array. - * This implementation is not synchronized and should only be used when - * thread-safety is not required. This implementation makes sense when the - * main operations performed are iterating over the cells and adding cells - * (especially if insertion is in sorted order). - */ -public class ArrayBackedSortedColumns extends ColumnFamily -{ - private static final Cell[] EMPTY_ARRAY = new Cell[0]; - private static final int MINIMAL_CAPACITY = 10; - - private final boolean reversed; - - private DeletionInfo deletionInfo; - private Cell[] cells; - private int size; - private int sortedSize; - private volatile boolean isSorted; - - public static final ColumnFamily.Factory factory = new Factory() - { - public ArrayBackedSortedColumns create(CFMetaData metadata, boolean insertReversed, int initialCapacity) - { - return new ArrayBackedSortedColumns(metadata, insertReversed, initialCapacity == 0 ? EMPTY_ARRAY : new Cell[initialCapacity], 0, 0); - } - }; - - private ArrayBackedSortedColumns(CFMetaData metadata, boolean reversed, Cell[] cells, int size, int sortedSize) - { - super(metadata); - this.reversed = reversed; - this.deletionInfo = DeletionInfo.live(); - this.cells = cells; - this.size = size; - this.sortedSize = sortedSize; - this.isSorted = size == sortedSize; - } - - protected ArrayBackedSortedColumns(CFMetaData metadata, boolean reversed) - { - this(metadata, reversed, EMPTY_ARRAY, 0, 0); - } - - private ArrayBackedSortedColumns(ArrayBackedSortedColumns original) - { - super(original.metadata); - this.reversed = original.reversed; - this.deletionInfo = DeletionInfo.live(); // this is INTENTIONALLY not set to original.deletionInfo. - this.cells = Arrays.copyOf(original.cells, original.size); - this.size = original.size; - this.sortedSize = original.sortedSize; - this.isSorted = original.isSorted; - } - - public static ArrayBackedSortedColumns localCopy(ColumnFamily original, AbstractAllocator allocator) - { - ArrayBackedSortedColumns copy = new ArrayBackedSortedColumns(original.metadata, false, new Cell[original.getColumnCount()], 0, 0); - for (Cell cell : original) - copy.internalAdd(cell.localCopy(original.metadata, allocator)); - copy.sortedSize = copy.size; // internalAdd doesn't update sortedSize. - copy.delete(original); - return copy; - } - - public ColumnFamily.Factory getFactory() - { - return factory; - } - - public ColumnFamily cloneMe() - { - return new ArrayBackedSortedColumns(this); - } - - public boolean isInsertReversed() - { - return reversed; - } - - public BatchRemoveIterator batchRemoveIterator() - { - maybeSortCells(); - - return new BatchRemoveIterator() - { - private final Iterator iter = iterator(); - private BitSet removedIndexes = new BitSet(size); - private int idx = -1; - private boolean shouldCallNext = false; - private boolean isCommitted = false; - private boolean removedAnything = false; - - public void commit() - { - if (isCommitted) - throw new IllegalStateException(); - isCommitted = true; - - if (!removedAnything) - return; - - int retainedCount = 0; - int clearIdx, setIdx = -1; - - // shift all [clearIdx, setIdx) segments to the left, skipping any removed columns - while (true) - { - clearIdx = removedIndexes.nextClearBit(setIdx + 1); - if (clearIdx >= size) - break; // nothing left to retain - - setIdx = removedIndexes.nextSetBit(clearIdx + 1); - if (setIdx < 0) - setIdx = size; // no removals past retainIdx - copy all remaining cells - - if (retainedCount != clearIdx) - System.arraycopy(cells, clearIdx, cells, retainedCount, setIdx - clearIdx); - - retainedCount += (setIdx - clearIdx); - } - - for (int i = retainedCount; i < size; i++) - cells[i] = null; - - size = sortedSize = retainedCount; - } - - public boolean hasNext() - { - return iter.hasNext(); - } - - public Cell next() - { - idx++; - shouldCallNext = false; - return iter.next(); - } - - public void remove() - { - if (shouldCallNext) - throw new IllegalStateException(); - - removedIndexes.set(reversed ? size - idx - 1 : idx); - removedAnything = true; - shouldCallNext = true; - } - }; - } - - private Comparator internalComparator() - { - return reversed ? getComparator().reverseComparator() : getComparator(); - } - - private void maybeSortCells() - { - if (!isSorted) - sortCells(); - } - - /** - * synchronized so that concurrent (read-only) accessors don't mess the internal state. - */ - private synchronized void sortCells() - { - if (isSorted) - return; // Just sorted by a previous call - - Comparator comparator = reversed - ? getComparator().columnReverseComparator() - : getComparator().columnComparator(false); - - // Sort the unsorted segment - will still potentially contain duplicate (non-reconciled) cells - Arrays.sort(cells, sortedSize, size, comparator); - - // Determine the merge start position for that segment - int pos = binarySearch(0, sortedSize, cells[sortedSize].name(), internalComparator()); - if (pos < 0) - pos = -pos - 1; - - // Copy [pos, lastSortedCellIndex] cells into a separate array - Cell[] leftCopy = pos == sortedSize - ? EMPTY_ARRAY - : Arrays.copyOfRange(cells, pos, sortedSize); - - // Store the beginning (inclusive) and the end (exclusive) indexes of the right segment - int rightStart = sortedSize; - int rightEnd = size; - - // 'Trim' the sizes to what's left without the leftCopy - size = sortedSize = pos; - - // Merge the cells from both segments. When adding from the left segment we can rely on it not having any - // duplicate cells, and thus omit the comparison with the previously entered cell - we'll never need to reconcile. - int l = 0, r = rightStart; - while (l < leftCopy.length && r < rightEnd) - { - int cmp = comparator.compare(leftCopy[l], cells[r]); - if (cmp < 0) - append(leftCopy[l++]); - else if (cmp == 0) - append(leftCopy[l++].reconcile(cells[r++])); - else - appendOrReconcile(cells[r++]); - } - while (l < leftCopy.length) - append(leftCopy[l++]); - while (r < rightEnd) - appendOrReconcile(cells[r++]); - - // Nullify the remainder of the array (in case we had duplicate cells that got reconciled) - for (int i = size; i < rightEnd; i++) - cells[i] = null; - - // Fully sorted at this point - isSorted = true; - } - - private void appendOrReconcile(Cell cell) - { - if (size > 0 && cells[size - 1].name().equals(cell.name())) - reconcileWith(size - 1, cell); - else - append(cell); - } - - private void append(Cell cell) - { - cells[size] = cell; - size++; - sortedSize++; - } - - public Cell getColumn(CellName name) - { - maybeSortCells(); - int pos = binarySearch(name); - return pos >= 0 ? cells[pos] : null; - } - - /** - * Adds a cell, assuming that: - * - it's non-gc-able (if a tombstone) or not a tombstone - * - it has a more recent timestamp than any partition/range tombstone shadowing it - * - it sorts *strictly after* the current-last cell in the array. - */ - public void maybeAppendColumn(Cell cell, DeletionInfo.InOrderTester tester, int gcBefore) - { - if (cell.getLocalDeletionTime() >= gcBefore && !tester.isDeleted(cell)) - appendColumn(cell); - } - - /** - * Adds a cell, assuming that it sorts *strictly after* the current-last cell in the array. - */ - public void appendColumn(Cell cell) - { - internalAdd(cell); - sortedSize++; - } - - public void addColumn(Cell cell) - { - if (size == 0) - { - internalAdd(cell); - sortedSize++; - return; - } - - if (!isSorted) - { - internalAdd(cell); - return; - } - - int c = internalComparator().compare(cells[size - 1].name(), cell.name()); - if (c < 0) - { - // Append to the end - internalAdd(cell); - sortedSize++; - } - else if (c == 0) - { - // Resolve against the last cell - reconcileWith(size - 1, cell); - } - else - { - int pos = binarySearch(cell.name()); - if (pos >= 0) // Reconcile with an existing cell - { - reconcileWith(pos, cell); - } - else - { - internalAdd(cell); // Append to the end, making cells unsorted from now on - isSorted = false; - } - } - } - - public void addAll(ColumnFamily other) - { - delete(other.deletionInfo()); - - if (!other.hasColumns()) - return; - - // In reality, with ABSC being the only remaining container (aside from ABTC), other will aways be ABSC. - if (size == 0 && other instanceof ArrayBackedSortedColumns) - { - fastAddAll((ArrayBackedSortedColumns) other); - } - else - { - Iterator iterator = reversed ? other.reverseIterator() : other.iterator(); - while (iterator.hasNext()) - addColumn(iterator.next()); - } - } - - // Fast path, when this ABSC is empty. - private void fastAddAll(ArrayBackedSortedColumns other) - { - if (other.isInsertReversed() == isInsertReversed()) - { - cells = Arrays.copyOf(other.cells, other.cells.length); - size = other.size; - sortedSize = other.sortedSize; - isSorted = other.isSorted; - } - else - { - if (cells.length < other.getColumnCount()) - cells = new Cell[Math.max(MINIMAL_CAPACITY, other.getColumnCount())]; - Iterator iterator = reversed ? other.reverseIterator() : other.iterator(); - while (iterator.hasNext()) - cells[size++] = iterator.next(); - sortedSize = size; - isSorted = true; - } - } - - /** - * Add a cell to the array, 'resizing' it first if necessary (if it doesn't fit). - */ - private void internalAdd(Cell cell) - { - if (cells.length == size) - cells = Arrays.copyOf(cells, Math.max(MINIMAL_CAPACITY, size * 3 / 2 + 1)); - cells[size++] = cell; - } - - /** - * Remove the cell at a given index, shifting the rest of the array to the left if needed. - * Please note that we mostly remove from the end, so the shifting should be rare. - */ - private void internalRemove(int index) - { - int moving = size - index - 1; - if (moving > 0) - System.arraycopy(cells, index + 1, cells, index, moving); - cells[--size] = null; - } - - /** - * Reconcile with a cell at position i. - * Assume that i is a valid position. - */ - private void reconcileWith(int i, Cell cell) - { - cells[i] = cell.reconcile(cells[i]); - } - - private int binarySearch(CellName name) - { - return binarySearch(0, size, name, internalComparator()); - } - - /** - * Simple binary search for a given cell name. - * The return value has the exact same meaning that the one of Collections.binarySearch(). - * (We don't use Collections.binarySearch() directly because it would require us to create - * a fake Cell (as well as an Cell comparator) to do the search, which is ugly. - */ - private int binarySearch(int fromIndex, int toIndex, Composite name, Comparator comparator) - { - int low = fromIndex; - int mid = toIndex; - int high = mid - 1; - int result = -1; - while (low <= high) - { - mid = (low + high) >> 1; - if ((result = comparator.compare(name, cells[mid].name())) > 0) - low = mid + 1; - else if (result == 0) - return mid; - else - high = mid - 1; - } - return -mid - (result < 0 ? 1 : 2); - } - - public Collection getSortedColumns() - { - return new CellCollection(reversed); - } - - public Collection getReverseSortedColumns() - { - return new CellCollection(!reversed); - } - - public int getColumnCount() - { - maybeSortCells(); - return size; - } - - public boolean hasColumns() - { - return size > 0; - } - - public void clear() - { - setDeletionInfo(DeletionInfo.live()); - for (int i = 0; i < size; i++) - cells[i] = null; - size = sortedSize = 0; - isSorted = true; - } - - public DeletionInfo deletionInfo() - { - return deletionInfo; - } - - public void delete(DeletionTime delTime) - { - deletionInfo.add(delTime); - } - - public void delete(DeletionInfo newInfo) - { - deletionInfo.add(newInfo); - } - - protected void delete(RangeTombstone tombstone) - { - deletionInfo.add(tombstone, getComparator()); - } - - public void setDeletionInfo(DeletionInfo newInfo) - { - deletionInfo = newInfo; - } - - /** - * Purges any tombstones with a local deletion time before gcBefore. - * @param gcBefore a timestamp (in seconds) before which tombstones should be purged - */ - public void purgeTombstones(int gcBefore) - { - deletionInfo.purge(gcBefore); - } - - public Iterable getColumnNames() - { - return Iterables.transform(new CellCollection(false), new Function() - { - public CellName apply(Cell cell) - { - return cell.name(); - } - }); - } - - public Iterator iterator(ColumnSlice[] slices) - { - maybeSortCells(); - return slices.length == 1 - ? slice(slices[0], reversed, null) - : new SlicesIterator(slices, reversed); - } - - public Iterator reverseIterator(ColumnSlice[] slices) - { - maybeSortCells(); - return slices.length == 1 - ? slice(slices[0], !reversed, null) - : new SlicesIterator(slices, !reversed); - } - - public SearchIterator searchIterator() - { - maybeSortCells(); - - return new SearchIterator() - { - // the first index that we could find the next key at, i.e. one larger - // than the last key's location - private int i = 0; - - // We assume a uniform distribution of keys, - // so we keep track of how many keys were skipped to satisfy last lookup, and only look at twice that - // many keys for next lookup initially, extending to whole range only if we couldn't find it in that subrange - private int range = size / 2; - - public boolean hasNext() - { - return i < size; - } - - public Cell next(CellName name) - { - if (!isSorted || !hasNext()) - throw new IllegalStateException(); - - // optimize for runs of sequential matches, as in CollationController - // checking to see if we've found the desired cells yet (CASSANDRA-6933) - int c = metadata.comparator.compare(name, cells[i].name()); - if (c <= 0) - return c < 0 ? null : cells[i++]; - - // use range to manually force a better bsearch "pivot" by breaking it into two calls: - // first for i..i+range, then i+range..size if necessary. - // https://issues.apache.org/jira/browse/CASSANDRA-6933?focusedCommentId=13958264&page=com.atlassian.jira.plugin.system.issuetabpanels:comment-tabpanel#comment-13958264 - int limit = Math.min(size, i + range); - int i2 = binarySearch(i + 1, limit, name, internalComparator()); - if (-1 - i2 == limit) - i2 = binarySearch(limit, size, name, internalComparator()); - // i2 can't be zero since we already checked cells[i] above - if (i2 > 0) - { - range = i2 - i; - i = i2 + 1; - return cells[i2]; - } - i2 = -1 - i2; - range = i2 - i; - i = i2; - return null; - } - }; - } - - private class SlicesIterator extends AbstractIterator - { - private final ColumnSlice[] slices; - private final boolean invert; - - private int idx = 0; - private int previousSliceEnd; - private Iterator currentSlice; - - public SlicesIterator(ColumnSlice[] slices, boolean invert) - { - this.slices = slices; - this.invert = invert; - previousSliceEnd = invert ? size : 0; - } - - protected Cell computeNext() - { - if (currentSlice == null) - { - if (idx >= slices.length) - return endOfData(); - currentSlice = slice(slices[idx++], invert, this); - } - - if (currentSlice.hasNext()) - return currentSlice.next(); - - currentSlice = null; - return computeNext(); - } - } - - /** - * @return a sub-range of our cells as an Iterator, between the provided composites (inclusive) - * - * @param slice The slice with the inclusive start and finish bounds - * @param invert If the sort order of our collection is opposite to the desired sort order of the result; - * this results in swapping the start/finish (since they are provided based on the desired - * sort order, not our sort order), to normalise to our sort order, and a backwards iterator is returned - * @param iter If this slice is part of a multi-slice, the iterator will be updated to ensure cells are visited only once - */ - private Iterator slice(ColumnSlice slice, boolean invert, SlicesIterator iter) - { - Composite start = invert ? slice.finish : slice.start; - Composite finish = invert ? slice.start : slice.finish; - - int lowerBound = 0, upperBound = size; - if (iter != null) - { - if (invert) - upperBound = iter.previousSliceEnd; - else - lowerBound = iter.previousSliceEnd; - } - - if (!start.isEmpty()) - { - lowerBound = binarySearch(lowerBound, upperBound, start, internalComparator()); - if (lowerBound < 0) - lowerBound = -lowerBound - 1; - } - - if (!finish.isEmpty()) - { - upperBound = binarySearch(lowerBound, upperBound, finish, internalComparator()); - upperBound = upperBound < 0 - ? -upperBound - 1 - : upperBound + 1; // upperBound is exclusive for the iterators - } - - // If we're going backwards (wrt our sort order) we store the startIdx and use it as our upper bound next round - if (iter != null) - iter.previousSliceEnd = invert ? lowerBound : upperBound; - - return invert - ? new BackwardsCellIterator(lowerBound, upperBound) - : new ForwardsCellIterator(lowerBound, upperBound); - } - - private final class BackwardsCellIterator implements Iterator - { - private int idx, end; - private boolean shouldCallNext = true; - - // lowerBound inclusive, upperBound exclusive - private BackwardsCellIterator(int lowerBound, int upperBound) - { - idx = upperBound - 1; - end = lowerBound - 1; - } - - public boolean hasNext() - { - return idx > end; - } - - public Cell next() - { - try - { - shouldCallNext = false; - return cells[idx--]; - } - catch (ArrayIndexOutOfBoundsException e) - { - NoSuchElementException ne = new NoSuchElementException(e.getMessage()); - ne.initCause(e); - throw ne; - } - } - - public void remove() - { - if (shouldCallNext) - throw new IllegalStateException(); - shouldCallNext = true; - internalRemove(idx + 1); - sortedSize--; - } - } - - private final class ForwardsCellIterator implements Iterator - { - private int idx, end; - private boolean shouldCallNext = true; - - // lowerBound inclusive, upperBound exclusive - private ForwardsCellIterator(int lowerBound, int upperBound) - { - idx = lowerBound; - end = upperBound; - } - - public boolean hasNext() - { - return idx < end; - } - - public Cell next() - { - try - { - shouldCallNext = false; - return cells[idx++]; - } - catch (ArrayIndexOutOfBoundsException e) - { - NoSuchElementException ne = new NoSuchElementException(e.getMessage()); - ne.initCause(e); - throw ne; - } - } - - public void remove() - { - if (shouldCallNext) - throw new IllegalStateException(); - shouldCallNext = true; - internalRemove(--idx); - sortedSize--; - end--; - } - } - - private final class CellCollection extends AbstractCollection - { - private final boolean invert; - - private CellCollection(boolean invert) - { - this.invert = invert; - } - - public int size() - { - return getColumnCount(); - } - - public Iterator iterator() - { - maybeSortCells(); - return invert - ? new BackwardsCellIterator(0, size) - : new ForwardsCellIterator(0, size); - } - } -} diff --git a/src/java/org/apache/cassandra/db/AtomDeserializer.java b/src/java/org/apache/cassandra/db/AtomDeserializer.java deleted file mode 100644 index 74f1946347..0000000000 --- a/src/java/org/apache/cassandra/db/AtomDeserializer.java +++ /dev/null @@ -1,128 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.apache.cassandra.db; - -import java.io.DataInput; -import java.io.IOException; - -import org.apache.cassandra.db.composites.CellName; -import org.apache.cassandra.db.composites.CellNameType; -import org.apache.cassandra.db.composites.Composite; -import org.apache.cassandra.io.sstable.format.Version; - -/** - * Helper class to deserialize OnDiskAtom efficiently. - * - * More precisely, this class is used by the low-level readers - * (IndexedSliceReader and SSTableNamesIterator) to ensure we don't - * do more work than necessary (i.e. we don't allocate/deserialize - * objects for things we don't care about). - */ -public class AtomDeserializer -{ - private final CellNameType type; - private final CellNameType.Deserializer nameDeserializer; - private final DataInput in; - private final ColumnSerializer.Flag flag; - private final int expireBefore; - private final Version version; - - // The "flag" for the next name (which correspond to the "masks" in ColumnSerializer) if it has been - // read already, Integer.MIN_VALUE otherwise; - private int nextFlags = Integer.MIN_VALUE; - - public AtomDeserializer(CellNameType type, DataInput in, ColumnSerializer.Flag flag, int expireBefore, Version version) - { - this.type = type; - this.nameDeserializer = type.newDeserializer(in); - this.in = in; - this.flag = flag; - this.expireBefore = expireBefore; - this.version = version; - } - - /** - * Whether or not there is more atom to read. - */ - public boolean hasNext() throws IOException - { - return nameDeserializer.hasNext(); - } - - /** - * Whether or not some atom has been read but not processed (neither readNext() nor - * skipNext() has been called for that atom) yet. - */ - public boolean hasUnprocessed() throws IOException - { - return nameDeserializer.hasUnprocessed(); - } - - /** - * Compare the provided composite to the next atom to read on disk. - * - * This will not read/deserialize the whole atom but only what is necessary for the - * comparison. Whenever we know what to do with this atom (read it or skip it), - * readNext or skipNext should be called. - */ - public int compareNextTo(Composite composite) throws IOException - { - return nameDeserializer.compareNextTo(composite); - } - - /** - * Returns whether the next atom is a range tombstone or not. - * - * Please note that this should only be called after compareNextTo() has been called. - */ - public boolean nextIsRangeTombstone() throws IOException - { - nextFlags = in.readUnsignedByte(); - return (nextFlags & ColumnSerializer.RANGE_TOMBSTONE_MASK) != 0; - } - - /** - * Returns the next atom. - */ - public OnDiskAtom readNext() throws IOException - { - Composite name = nameDeserializer.readNext(); - assert !name.isEmpty(); // This would imply hasNext() hasn't been called - - nextFlags = nextFlags == Integer.MIN_VALUE ? in.readUnsignedByte() : nextFlags; - OnDiskAtom atom = (nextFlags & ColumnSerializer.RANGE_TOMBSTONE_MASK) != 0 - ? type.rangeTombstoneSerializer().deserializeBody(in, name, version) - : type.columnSerializer().deserializeColumnBody(in, (CellName)name, nextFlags, flag, expireBefore); - nextFlags = Integer.MIN_VALUE; - return atom; - } - - /** - * Skips the next atom. - */ - public void skipNext() throws IOException - { - nameDeserializer.skipNext(); - nextFlags = nextFlags == Integer.MIN_VALUE ? in.readUnsignedByte() : nextFlags; - if ((nextFlags & ColumnSerializer.RANGE_TOMBSTONE_MASK) != 0) - type.rangeTombstoneSerializer().skipBody(in, version); - else - type.columnSerializer().skipColumnBody(in, nextFlags); - nextFlags = Integer.MIN_VALUE; - } -} diff --git a/src/java/org/apache/cassandra/db/AtomicBTreeColumns.java b/src/java/org/apache/cassandra/db/AtomicBTreeColumns.java deleted file mode 100644 index 9ef0c149f6..0000000000 --- a/src/java/org/apache/cassandra/db/AtomicBTreeColumns.java +++ /dev/null @@ -1,578 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.apache.cassandra.db; - -import java.util.AbstractCollection; -import java.util.ArrayList; -import java.util.Collection; -import java.util.Comparator; -import java.util.Iterator; -import java.util.List; -import java.util.concurrent.atomic.AtomicIntegerFieldUpdater; -import java.util.concurrent.atomic.AtomicReferenceFieldUpdater; - -import com.google.common.base.Function; -import com.google.common.base.Functions; -import com.google.common.collect.AbstractIterator; -import com.google.common.collect.Iterators; - -import org.apache.cassandra.config.CFMetaData; -import org.apache.cassandra.db.composites.CellName; -import org.apache.cassandra.db.composites.Composite; -import org.apache.cassandra.db.filter.ColumnSlice; -import org.apache.cassandra.db.marshal.BytesType; -import org.apache.cassandra.utils.*; -import org.apache.cassandra.utils.SearchIterator; -import org.apache.cassandra.utils.btree.BTree; -import org.apache.cassandra.utils.btree.BTreeSearchIterator; -import org.apache.cassandra.utils.btree.UpdateFunction; -import org.apache.cassandra.utils.concurrent.Locks; -import org.apache.cassandra.utils.concurrent.OpOrder; -import org.apache.cassandra.utils.memory.HeapAllocator; -import org.apache.cassandra.utils.memory.MemtableAllocator; -import org.apache.cassandra.utils.memory.NativePool; - -import static org.apache.cassandra.db.index.SecondaryIndexManager.Updater; - -/** - * A thread-safe and atomic ISortedColumns implementation. - * Operations (in particular addAll) on this implemenation are atomic and - * isolated (in the sense of ACID). Typically a addAll is guaranteed that no - * other thread can see the state where only parts but not all columns have - * been added. - *

- * WARNING: removing element through getSortedColumns().iterator() is *not* supported - *

- */ -public class AtomicBTreeColumns extends ColumnFamily -{ - static final long EMPTY_SIZE = ObjectSizes.measure(new AtomicBTreeColumns(CFMetaData.denseCFMetaData("keyspace", "table", BytesType.instance), null)) - + ObjectSizes.measure(new Holder(null, null)); - - // Reserved values for wasteTracker field. These values must not be consecutive (see avoidReservedValues) - private static final int TRACKER_NEVER_WASTED = 0; - private static final int TRACKER_PESSIMISTIC_LOCKING = Integer.MAX_VALUE; - - // The granularity with which we track wasted allocation/work; we round up - private static final int ALLOCATION_GRANULARITY_BYTES = 1024; - // The number of bytes we have to waste in excess of our acceptable realtime rate of waste (defined below) - private static final long EXCESS_WASTE_BYTES = 10 * 1024 * 1024L; - private static final int EXCESS_WASTE_OFFSET = (int) (EXCESS_WASTE_BYTES / ALLOCATION_GRANULARITY_BYTES); - // Note this is a shift, because dividing a long time and then picking the low 32 bits doesn't give correct rollover behavior - private static final int CLOCK_SHIFT = 17; - // CLOCK_GRANULARITY = 1^9ns >> CLOCK_SHIFT == 132us == (1/7.63)ms - - /** - * (clock + allocation) granularity are combined to give us an acceptable (waste) allocation rate that is defined by - * the passage of real time of ALLOCATION_GRANULARITY_BYTES/CLOCK_GRANULARITY, or in this case 7.63Kb/ms, or 7.45Mb/s - * - * in wasteTracker we maintain within EXCESS_WASTE_OFFSET before the current time; whenever we waste bytes - * we increment the current value if it is within this window, and set it to the min of the window plus our waste - * otherwise. - */ - private volatile int wasteTracker = TRACKER_NEVER_WASTED; - - private static final AtomicIntegerFieldUpdater wasteTrackerUpdater = AtomicIntegerFieldUpdater.newUpdater(AtomicBTreeColumns.class, "wasteTracker"); - - private static final Function NAME = new Function() - { - public CellName apply(Cell column) - { - return column.name(); - } - }; - - public static final Factory factory = new Factory() - { - public AtomicBTreeColumns create(CFMetaData metadata, boolean insertReversed, int initialCapacity) - { - if (insertReversed) - throw new IllegalArgumentException(); - return new AtomicBTreeColumns(metadata); - } - }; - - private static final DeletionInfo LIVE = DeletionInfo.live(); - // This is a small optimization: DeletionInfo is mutable, but we know that we will always copy it in that class, - // so we can safely alias one DeletionInfo.live() reference and avoid some allocations. - private static final Holder EMPTY = new Holder(BTree.empty(), LIVE); - - private volatile Holder ref; - - private static final AtomicReferenceFieldUpdater refUpdater = AtomicReferenceFieldUpdater.newUpdater(AtomicBTreeColumns.class, Holder.class, "ref"); - - private AtomicBTreeColumns(CFMetaData metadata) - { - this(metadata, EMPTY); - } - - private AtomicBTreeColumns(CFMetaData metadata, Holder holder) - { - super(metadata); - this.ref = holder; - } - - public Factory getFactory() - { - return factory; - } - - public ColumnFamily cloneMe() - { - return new AtomicBTreeColumns(metadata, ref); - } - - public DeletionInfo deletionInfo() - { - return ref.deletionInfo; - } - - public void delete(DeletionTime delTime) - { - delete(new DeletionInfo(delTime)); - } - - protected void delete(RangeTombstone tombstone) - { - delete(new DeletionInfo(tombstone, getComparator())); - } - - public SearchIterator searchIterator() - { - return new BTreeSearchIterator<>(ref.tree, asymmetricComparator()); - } - - public void delete(DeletionInfo info) - { - if (info.isLive()) - return; - - // Keeping deletion info for max markedForDeleteAt value - while (true) - { - Holder current = ref; - DeletionInfo curDelInfo = current.deletionInfo; - DeletionInfo newDelInfo = info.mayModify(curDelInfo) ? curDelInfo.copy().add(info) : curDelInfo; - if (refUpdater.compareAndSet(this, current, current.with(newDelInfo))) - break; - } - } - - public void setDeletionInfo(DeletionInfo newInfo) - { - ref = ref.with(newInfo); - } - - public void purgeTombstones(int gcBefore) - { - while (true) - { - Holder current = ref; - if (!current.deletionInfo.hasPurgeableTombstones(gcBefore)) - break; - - DeletionInfo purgedInfo = current.deletionInfo.copy(); - purgedInfo.purge(gcBefore); - if (refUpdater.compareAndSet(this, current, current.with(purgedInfo))) - break; - } - } - - /** - * This is only called by Memtable.resolve, so only AtomicBTreeColumns needs to implement it. - * - * @return the difference in size seen after merging the given columns - */ - public Pair addAllWithSizeDelta(final ColumnFamily cm, MemtableAllocator allocator, OpOrder.Group writeOp, Updater indexer) - { - ColumnUpdater updater = new ColumnUpdater(this, cm.metadata, allocator, writeOp, indexer); - DeletionInfo inputDeletionInfoCopy = null; - - boolean monitorOwned = false; - try - { - if (usePessimisticLocking()) - { - Locks.monitorEnterUnsafe(this); - monitorOwned = true; - } - while (true) - { - Holder current = ref; - updater.ref = current; - updater.reset(); - - DeletionInfo deletionInfo; - if (cm.deletionInfo().mayModify(current.deletionInfo)) - { - if (inputDeletionInfoCopy == null) - inputDeletionInfoCopy = cm.deletionInfo().copy(HeapAllocator.instance); - - deletionInfo = current.deletionInfo.copy().add(inputDeletionInfoCopy); - updater.allocated(deletionInfo.unsharedHeapSize() - current.deletionInfo.unsharedHeapSize()); - } - else - { - deletionInfo = current.deletionInfo; - } - - Object[] tree = BTree.update(current.tree, metadata.comparator.columnComparator(Memtable.MEMORY_POOL instanceof NativePool), cm, cm.getColumnCount(), true, updater); - - if (tree != null && refUpdater.compareAndSet(this, current, new Holder(tree, deletionInfo))) - { - indexer.updateRowLevelIndexes(); - updater.finish(); - return Pair.create(updater.dataSize, updater.colUpdateTimeDelta); - } - else if (!monitorOwned) - { - boolean shouldLock = usePessimisticLocking(); - if (!shouldLock) - { - shouldLock = updateWastedAllocationTracker(updater.heapSize); - } - if (shouldLock) - { - Locks.monitorEnterUnsafe(this); - monitorOwned = true; - } - } - } - } - finally - { - if (monitorOwned) - Locks.monitorExitUnsafe(this); - } - } - - boolean usePessimisticLocking() - { - return wasteTracker == TRACKER_PESSIMISTIC_LOCKING; - } - - /** - * Update the wasted allocation tracker state based on newly wasted allocation information - * - * @param wastedBytes the number of bytes wasted by this thread - * @return true if the caller should now proceed with pessimistic locking because the waste limit has been reached - */ - private boolean updateWastedAllocationTracker(long wastedBytes) { - // Early check for huge allocation that exceeds the limit - if (wastedBytes < EXCESS_WASTE_BYTES) - { - // We round up to ensure work < granularity are still accounted for - int wastedAllocation = ((int) (wastedBytes + ALLOCATION_GRANULARITY_BYTES - 1)) / ALLOCATION_GRANULARITY_BYTES; - - int oldTrackerValue; - while (TRACKER_PESSIMISTIC_LOCKING != (oldTrackerValue = wasteTracker)) - { - // Note this time value has an arbitrary offset, but is a constant rate 32 bit counter (that may wrap) - int time = (int) (System.nanoTime() >>> CLOCK_SHIFT); - int delta = oldTrackerValue - time; - if (oldTrackerValue == TRACKER_NEVER_WASTED || delta >= 0 || delta < -EXCESS_WASTE_OFFSET) - delta = -EXCESS_WASTE_OFFSET; - delta += wastedAllocation; - if (delta >= 0) - break; - if (wasteTrackerUpdater.compareAndSet(this, oldTrackerValue, avoidReservedValues(time + delta))) - return false; - } - } - // We have definitely reached our waste limit so set the state if it isn't already - wasteTrackerUpdater.set(this, TRACKER_PESSIMISTIC_LOCKING); - // And tell the caller to proceed with pessimistic locking - return true; - } - - private static int avoidReservedValues(int wasteTracker) - { - if (wasteTracker == TRACKER_NEVER_WASTED || wasteTracker == TRACKER_PESSIMISTIC_LOCKING) - return wasteTracker + 1; - return wasteTracker; - } - - // no particular reason not to implement these next methods, we just haven't needed them yet - - public void addColumn(Cell column) - { - throw new UnsupportedOperationException(); - } - - public void maybeAppendColumn(Cell cell, DeletionInfo.InOrderTester tester, int gcBefore) - { - throw new UnsupportedOperationException(); - } - - public void appendColumn(Cell cell) - { - throw new UnsupportedOperationException(); - } - - public void addAll(ColumnFamily cf) - { - throw new UnsupportedOperationException(); - } - - public void clear() - { - throw new UnsupportedOperationException(); - } - - public Cell getColumn(CellName name) - { - return (Cell) BTree.find(ref.tree, asymmetricComparator(), name); - } - - private Comparator asymmetricComparator() - { - return metadata.comparator.asymmetricColumnComparator(Memtable.MEMORY_POOL instanceof NativePool); - } - - public Iterable getColumnNames() - { - return collection(false, NAME); - } - - public Collection getSortedColumns() - { - return collection(true, Functions.identity()); - } - - public Collection getReverseSortedColumns() - { - return collection(false, Functions.identity()); - } - - private Collection collection(final boolean forwards, final Function f) - { - final Holder ref = this.ref; - return new AbstractCollection() - { - public Iterator iterator() - { - return Iterators.transform(BTree.slice(ref.tree, forwards), f); - } - - public int size() - { - return BTree.slice(ref.tree, true).count(); - } - }; - } - - public int getColumnCount() - { - return BTree.slice(ref.tree, true).count(); - } - - public boolean hasColumns() - { - return !BTree.isEmpty(ref.tree); - } - - public Iterator iterator(ColumnSlice[] slices) - { - return slices.length == 1 - ? slice(ref.tree, asymmetricComparator(), slices[0].start, slices[0].finish, true) - : new SliceIterator(ref.tree, asymmetricComparator(), true, slices); - } - - public Iterator reverseIterator(ColumnSlice[] slices) - { - return slices.length == 1 - ? slice(ref.tree, asymmetricComparator(), slices[0].finish, slices[0].start, false) - : new SliceIterator(ref.tree, asymmetricComparator(), false, slices); - } - - public boolean isInsertReversed() - { - return false; - } - - public BatchRemoveIterator batchRemoveIterator() - { - throw new UnsupportedOperationException(); - } - - private static final class Holder - { - final DeletionInfo deletionInfo; - // the btree of columns - final Object[] tree; - - Holder(Object[] tree, DeletionInfo deletionInfo) - { - this.tree = tree; - this.deletionInfo = deletionInfo; - } - - Holder with(DeletionInfo info) - { - return new Holder(this.tree, info); - } - } - - // the function we provide to the btree utilities to perform any column replacements - private static final class ColumnUpdater implements UpdateFunction - { - final AtomicBTreeColumns updating; - final CFMetaData metadata; - final MemtableAllocator allocator; - final OpOrder.Group writeOp; - final Updater indexer; - Holder ref; - long dataSize; - long heapSize; - long colUpdateTimeDelta = Long.MAX_VALUE; - final MemtableAllocator.DataReclaimer reclaimer; - List inserted; // TODO: replace with walk of aborted BTree - - private ColumnUpdater(AtomicBTreeColumns updating, CFMetaData metadata, MemtableAllocator allocator, OpOrder.Group writeOp, Updater indexer) - { - this.updating = updating; - this.allocator = allocator; - this.writeOp = writeOp; - this.indexer = indexer; - this.metadata = metadata; - this.reclaimer = allocator.reclaimer(); - } - - public Cell apply(Cell insert) - { - indexer.insert(insert); - insert = insert.localCopy(metadata, allocator, writeOp); - this.dataSize += insert.cellDataSize(); - this.heapSize += insert.unsharedHeapSizeExcludingData(); - if (inserted == null) - inserted = new ArrayList<>(); - inserted.add(insert); - return insert; - } - - public Cell apply(Cell existing, Cell update) - { - Cell reconciled = existing.reconcile(update); - indexer.update(existing, reconciled); - if (existing != reconciled) - { - reconciled = reconciled.localCopy(metadata, allocator, writeOp); - dataSize += reconciled.cellDataSize() - existing.cellDataSize(); - heapSize += reconciled.unsharedHeapSizeExcludingData() - existing.unsharedHeapSizeExcludingData(); - if (inserted == null) - inserted = new ArrayList<>(); - inserted.add(reconciled); - discard(existing); - //Getting the minimum delta for an update containing multiple columns - colUpdateTimeDelta = Math.min(Math.abs(existing.timestamp() - update.timestamp()), colUpdateTimeDelta); - } - return reconciled; - } - - protected void reset() - { - this.dataSize = 0; - this.heapSize = 0; - if (inserted != null) - { - for (Cell cell : inserted) - abort(cell); - inserted.clear(); - } - reclaimer.cancel(); - } - - protected void abort(Cell abort) - { - reclaimer.reclaimImmediately(abort); - } - - protected void discard(Cell discard) - { - reclaimer.reclaim(discard); - } - - public boolean abortEarly() - { - return updating.ref != ref; - } - - public void allocated(long heapSize) - { - this.heapSize += heapSize; - } - - protected void finish() - { - allocator.onHeap().allocate(heapSize, writeOp); - reclaimer.commit(); - } - } - - private static class SliceIterator extends AbstractIterator - { - private final Object[] btree; - private final boolean forwards; - private final Comparator comparator; - private final ColumnSlice[] slices; - - private int idx = 0; - private Iterator currentSlice; - - SliceIterator(Object[] btree, Comparator comparator, boolean forwards, ColumnSlice[] slices) - { - this.btree = btree; - this.comparator = comparator; - this.slices = slices; - this.forwards = forwards; - } - - protected Cell computeNext() - { - while (currentSlice != null || idx < slices.length) - { - if (currentSlice == null) - { - ColumnSlice slice = slices[idx++]; - if (forwards) - currentSlice = slice(btree, comparator, slice.start, slice.finish, true); - else - currentSlice = slice(btree, comparator, slice.finish, slice.start, false); - } - - if (currentSlice.hasNext()) - return currentSlice.next(); - - currentSlice = null; - } - - return endOfData(); - } - } - - private static Iterator slice(Object[] btree, Comparator comparator, Composite start, Composite finish, boolean forwards) - { - return BTree.slice(btree, - comparator, - start.isEmpty() ? null : start, - true, - finish.isEmpty() ? null : finish, - true, - forwards); - } -} diff --git a/src/java/org/apache/cassandra/db/BatchlogManager.java b/src/java/org/apache/cassandra/db/BatchlogManager.java index 6038475fa4..83a06545a4 100644 --- a/src/java/org/apache/cassandra/db/BatchlogManager.java +++ b/src/java/org/apache/cassandra/db/BatchlogManager.java @@ -38,6 +38,7 @@ import org.slf4j.LoggerFactory; import org.apache.cassandra.concurrent.DebuggableScheduledThreadPoolExecutor; import org.apache.cassandra.config.DatabaseDescriptor; import org.apache.cassandra.cql3.UntypedResultSet; +import org.apache.cassandra.db.partitions.PartitionUpdate; import org.apache.cassandra.db.compaction.CompactionManager; import org.apache.cassandra.db.marshal.UUIDType; import org.apache.cassandra.dht.Token; @@ -138,12 +139,12 @@ public class BatchlogManager implements BatchlogManagerMBean @VisibleForTesting static Mutation getBatchlogMutationFor(Collection mutations, UUID uuid, int version, long now) { - ColumnFamily cf = ArrayBackedSortedColumns.factory.create(SystemKeyspace.Batchlog); - CFRowAdder adder = new CFRowAdder(cf, SystemKeyspace.Batchlog.comparator.builder().build(), now); - adder.add("data", serializeMutations(mutations, version)) - .add("written_at", new Date(now / 1000)) - .add("version", version); - return new Mutation(SystemKeyspace.NAME, UUIDType.instance.decompose(uuid), cf); + return new RowUpdateBuilder(SystemKeyspace.Batchlog, now, uuid) + .clustering() + .add("data", serializeMutations(mutations, version)) + .add("written_at", new Date(now / 1000)) + .add("version", version) + .build(); } private static ByteBuffer serializeMutations(Collection mutations, int version) @@ -186,7 +187,7 @@ public class BatchlogManager implements BatchlogManagerMBean SystemKeyspace.NAME, SystemKeyspace.BATCHLOG, PAGE_SIZE), - id); + id); } cleanup(); @@ -196,8 +197,8 @@ public class BatchlogManager implements BatchlogManagerMBean private void deleteBatch(UUID id) { - Mutation mutation = new Mutation(SystemKeyspace.NAME, UUIDType.instance.decompose(id)); - mutation.delete(SystemKeyspace.BATCHLOG, FBUtilities.timestampMicros()); + Mutation mutation = new Mutation(SystemKeyspace.NAME, StorageService.getPartitioner().decorateKey(UUIDType.instance.decompose(id))); + mutation.add(PartitionUpdate.fullPartitionDelete(SystemKeyspace.Batchlog, mutation.key(), FBUtilities.timestampMicros(), FBUtilities.nowInSeconds())); mutation.apply(); } @@ -382,7 +383,7 @@ public class BatchlogManager implements BatchlogManagerMBean { Set liveEndpoints = new HashSet<>(); String ks = mutation.getKeyspaceName(); - Token tk = StorageService.getPartitioner().getToken(mutation.key()); + Token tk = mutation.key().getToken(); for (InetAddress endpoint : Iterables.concat(StorageService.instance.getNaturalEndpoints(ks, tk), StorageService.instance.getTokenMetadata().pendingEndpointsFor(tk, ks))) diff --git a/src/java/org/apache/cassandra/db/BufferCell.java b/src/java/org/apache/cassandra/db/BufferCell.java deleted file mode 100644 index a7d632d908..0000000000 --- a/src/java/org/apache/cassandra/db/BufferCell.java +++ /dev/null @@ -1,103 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.apache.cassandra.db; - -import java.nio.ByteBuffer; - -import org.apache.cassandra.config.CFMetaData; -import org.apache.cassandra.db.composites.CellName; -import org.apache.cassandra.db.composites.CellNames; -import org.apache.cassandra.utils.ByteBufferUtil; -import org.apache.cassandra.utils.ObjectSizes; -import org.apache.cassandra.utils.concurrent.OpOrder; -import org.apache.cassandra.utils.memory.AbstractAllocator; -import org.apache.cassandra.utils.memory.MemtableAllocator; - -public class BufferCell extends AbstractCell -{ - private static final long EMPTY_SIZE = ObjectSizes.measure(new BufferCell(CellNames.simpleDense(ByteBuffer.allocate(1)))); - - protected final CellName name; - protected final ByteBuffer value; - protected final long timestamp; - - BufferCell(CellName name) - { - this(name, ByteBufferUtil.EMPTY_BYTE_BUFFER); - } - - public BufferCell(CellName name, ByteBuffer value) - { - this(name, value, 0); - } - - public BufferCell(CellName name, ByteBuffer value, long timestamp) - { - assert name != null; - assert value != null; - - this.name = name; - this.value = value; - this.timestamp = timestamp; - } - - @Override - public Cell withUpdatedName(CellName newName) - { - return new BufferCell(newName, value, timestamp); - } - - @Override - public Cell withUpdatedTimestamp(long newTimestamp) - { - return new BufferCell(name, value, newTimestamp); - } - - @Override - public CellName name() { - return name; - } - - @Override - public ByteBuffer value() { - return value; - } - - @Override - public long timestamp() { - return timestamp; - } - - @Override - public long unsharedHeapSizeExcludingData() - { - return EMPTY_SIZE + name.unsharedHeapSizeExcludingData() + ObjectSizes.sizeOnHeapExcludingData(value); - } - - @Override - public Cell localCopy(CFMetaData metadata, AbstractAllocator allocator) - { - return new BufferCell(name.copy(metadata, allocator), allocator.clone(value), timestamp); - } - - @Override - public Cell localCopy(CFMetaData metadata, MemtableAllocator allocator, OpOrder.Group opGroup) - { - return allocator.clone(this, metadata, opGroup); - } -} diff --git a/src/java/org/apache/cassandra/db/BufferCounterCell.java b/src/java/org/apache/cassandra/db/BufferCounterCell.java deleted file mode 100644 index 827182a340..0000000000 --- a/src/java/org/apache/cassandra/db/BufferCounterCell.java +++ /dev/null @@ -1,176 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.apache.cassandra.db; - -import java.nio.ByteBuffer; -import java.security.MessageDigest; - -import org.apache.cassandra.config.CFMetaData; -import org.apache.cassandra.db.composites.CellName; -import org.apache.cassandra.db.composites.CellNameType; -import org.apache.cassandra.serializers.MarshalException; -import org.apache.cassandra.utils.FBUtilities; -import org.apache.cassandra.utils.concurrent.OpOrder; -import org.apache.cassandra.utils.memory.AbstractAllocator; -import org.apache.cassandra.utils.memory.MemtableAllocator; - -public class BufferCounterCell extends BufferCell implements CounterCell -{ - private final long timestampOfLastDelete; - - public BufferCounterCell(CellName name, ByteBuffer value, long timestamp) - { - this(name, value, timestamp, Long.MIN_VALUE); - } - - public BufferCounterCell(CellName name, ByteBuffer value, long timestamp, long timestampOfLastDelete) - { - super(name, value, timestamp); - this.timestampOfLastDelete = timestampOfLastDelete; - } - - public static CounterCell create(CellName name, ByteBuffer value, long timestamp, long timestampOfLastDelete, ColumnSerializer.Flag flag) - { - if (flag == ColumnSerializer.Flag.FROM_REMOTE || (flag == ColumnSerializer.Flag.LOCAL && contextManager.shouldClearLocal(value))) - value = contextManager.clearAllLocal(value); - return new BufferCounterCell(name, value, timestamp, timestampOfLastDelete); - } - - // For use by tests of compatibility with pre-2.1 counter only. - public static CounterCell createLocal(CellName name, long value, long timestamp, long timestampOfLastDelete) - { - return new BufferCounterCell(name, contextManager.createLocal(value), timestamp, timestampOfLastDelete); - } - - @Override - public Cell withUpdatedName(CellName newName) - { - return new BufferCounterCell(newName, value, timestamp, timestampOfLastDelete); - } - - @Override - public long timestampOfLastDelete() - { - return timestampOfLastDelete; - } - - @Override - public long total() - { - return contextManager.total(value); - } - - @Override - public int cellDataSize() - { - // A counter column adds 8 bytes for timestampOfLastDelete to Cell. - return super.cellDataSize() + TypeSizes.NATIVE.sizeof(timestampOfLastDelete); - } - - @Override - public int serializedSize(CellNameType type, TypeSizes typeSizes) - { - return super.serializedSize(type, typeSizes) + typeSizes.sizeof(timestampOfLastDelete); - } - - @Override - public Cell diff(Cell cell) - { - return diffCounter(cell); - } - - /* - * We have to special case digest creation for counter column because - * we don't want to include the information about which shard of the - * context is a delta or not, since this information differs from node to - * node. - */ - @Override - public void updateDigest(MessageDigest digest) - { - digest.update(name().toByteBuffer().duplicate()); - // We don't take the deltas into account in a digest - contextManager.updateDigest(digest, value()); - - FBUtilities.updateWithLong(digest, timestamp); - FBUtilities.updateWithByte(digest, serializationFlags()); - FBUtilities.updateWithLong(digest, timestampOfLastDelete); - } - - @Override - public Cell reconcile(Cell cell) - { - return reconcileCounter(cell); - } - - @Override - public boolean hasLegacyShards() - { - return contextManager.hasLegacyShards(value); - } - - @Override - public CounterCell localCopy(CFMetaData metadata, AbstractAllocator allocator) - { - return new BufferCounterCell(name.copy(metadata, allocator), allocator.clone(value), timestamp, timestampOfLastDelete); - } - - @Override - public CounterCell localCopy(CFMetaData metadata, MemtableAllocator allocator, OpOrder.Group opGroup) - { - return allocator.clone(this, metadata, opGroup); - } - - @Override - public String getString(CellNameType comparator) - { - return String.format("%s:false:%s@%d!%d", - comparator.getString(name()), - contextManager.toString(value()), - timestamp(), - timestampOfLastDelete); - } - - @Override - public int serializationFlags() - { - return ColumnSerializer.COUNTER_MASK; - } - - @Override - public void validateFields(CFMetaData metadata) throws MarshalException - { - validateName(metadata); - // We cannot use the value validator as for other columns as the CounterColumnType validate a long, - // which is not the internal representation of counters - contextManager.validateContext(value()); - } - - @Override - public Cell markLocalToBeCleared() - { - ByteBuffer marked = contextManager.markLocalToBeCleared(value()); - return marked == value() ? this : new BufferCounterCell(name(), marked, timestamp(), timestampOfLastDelete); - } - - @Override - public boolean equals(Cell cell) - { - return super.equals(cell) && timestampOfLastDelete == ((CounterCell) cell).timestampOfLastDelete(); - } -} diff --git a/src/java/org/apache/cassandra/db/BufferCounterUpdateCell.java b/src/java/org/apache/cassandra/db/BufferCounterUpdateCell.java deleted file mode 100644 index f7df3eadff..0000000000 --- a/src/java/org/apache/cassandra/db/BufferCounterUpdateCell.java +++ /dev/null @@ -1,96 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.apache.cassandra.db; - -import java.nio.ByteBuffer; - -import org.apache.cassandra.config.CFMetaData; -import org.apache.cassandra.db.composites.CellName; -import org.apache.cassandra.db.composites.CellNameType; -import org.apache.cassandra.utils.ByteBufferUtil; -import org.apache.cassandra.utils.concurrent.OpOrder; -import org.apache.cassandra.utils.memory.AbstractAllocator; -import org.apache.cassandra.utils.memory.MemtableAllocator; - -public class BufferCounterUpdateCell extends BufferCell implements CounterUpdateCell -{ - public BufferCounterUpdateCell(CellName name, long value, long timestamp) - { - this(name, ByteBufferUtil.bytes(value), timestamp); - } - - public BufferCounterUpdateCell(CellName name, ByteBuffer value, long timestamp) - { - super(name, value, timestamp); - } - - @Override - public Cell withUpdatedName(CellName newName) - { - return new BufferCounterUpdateCell(newName, value, timestamp); - } - - public long delta() - { - return value().getLong(value.position()); - } - - @Override - public Cell diff(Cell cell) - { - // Diff is used during reads, but we should never read those columns - throw new UnsupportedOperationException("This operation is unsupported on CounterUpdateCell."); - } - - @Override - public Cell reconcile(Cell cell) - { - // No matter what the counter cell's timestamp is, a tombstone always takes precedence. See CASSANDRA-7346. - if (cell instanceof DeletedCell) - return cell; - - assert cell instanceof CounterUpdateCell : "Wrong class type."; - - // The only time this could happen is if a batch ships two increments for the same cell. Hence we simply sum the deltas. - return new BufferCounterUpdateCell(name, delta() + ((CounterUpdateCell) cell).delta(), Math.max(timestamp, cell.timestamp())); - } - - @Override - public int serializationFlags() - { - return ColumnSerializer.COUNTER_UPDATE_MASK; - } - - @Override - public Cell localCopy(CFMetaData metadata, AbstractAllocator allocator) - { - throw new UnsupportedOperationException(); - } - - @Override - public Cell localCopy(CFMetaData metadata, MemtableAllocator allocator, OpOrder.Group opGroup) - { - throw new UnsupportedOperationException(); - } - - @Override - public String getString(CellNameType comparator) - { - return String.format("%s:%s@%d", comparator.getString(name()), ByteBufferUtil.toLong(value), timestamp()); - } -} diff --git a/src/java/org/apache/cassandra/db/BufferDeletedCell.java b/src/java/org/apache/cassandra/db/BufferDeletedCell.java deleted file mode 100644 index a38f322bb2..0000000000 --- a/src/java/org/apache/cassandra/db/BufferDeletedCell.java +++ /dev/null @@ -1,118 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.apache.cassandra.db; - -import java.nio.ByteBuffer; -import java.security.MessageDigest; - -import org.apache.cassandra.config.CFMetaData; -import org.apache.cassandra.db.composites.CellName; -import org.apache.cassandra.serializers.MarshalException; -import org.apache.cassandra.utils.FBUtilities; -import org.apache.cassandra.utils.concurrent.OpOrder; -import org.apache.cassandra.utils.memory.AbstractAllocator; -import org.apache.cassandra.utils.ByteBufferUtil; -import org.apache.cassandra.utils.memory.MemtableAllocator; - -public class BufferDeletedCell extends BufferCell implements DeletedCell -{ - public BufferDeletedCell(CellName name, int localDeletionTime, long timestamp) - { - this(name, ByteBufferUtil.bytes(localDeletionTime), timestamp); - } - - public BufferDeletedCell(CellName name, ByteBuffer value, long timestamp) - { - super(name, value, timestamp); - } - - @Override - public Cell withUpdatedName(CellName newName) - { - return new BufferDeletedCell(newName, value, timestamp); - } - - @Override - public Cell withUpdatedTimestamp(long newTimestamp) - { - return new BufferDeletedCell(name, value, newTimestamp); - } - - @Override - public boolean isLive() - { - return false; - } - - @Override - public boolean isLive(long now) - { - return false; - } - - @Override - public int getLocalDeletionTime() - { - return value().getInt(value.position()); - } - - @Override - public Cell reconcile(Cell cell) - { - if (cell instanceof DeletedCell) - return super.reconcile(cell); - return cell.reconcile(this); - } - - @Override - public DeletedCell localCopy(CFMetaData metadata, AbstractAllocator allocator) - { - return new BufferDeletedCell(name.copy(metadata, allocator), allocator.clone(value), timestamp); - } - - @Override - public DeletedCell localCopy(CFMetaData metadata, MemtableAllocator allocator, OpOrder.Group opGroup) - { - return allocator.clone(this, metadata, opGroup); - } - - @Override - public int serializationFlags() - { - return ColumnSerializer.DELETION_MASK; - } - - @Override - public void validateFields(CFMetaData metadata) throws MarshalException - { - validateName(metadata); - if (value().remaining() != 4) - throw new MarshalException("A tombstone value should be 4 bytes long"); - if (getLocalDeletionTime() < 0) - throw new MarshalException("The local deletion time should not be negative"); - } - - @Override - public void updateDigest(MessageDigest digest) - { - digest.update(name().toByteBuffer().duplicate()); - - FBUtilities.updateWithLong(digest, timestamp()); - FBUtilities.updateWithByte(digest, serializationFlags()); - } -} diff --git a/src/java/org/apache/cassandra/db/BufferExpiringCell.java b/src/java/org/apache/cassandra/db/BufferExpiringCell.java deleted file mode 100644 index efb56d5727..0000000000 --- a/src/java/org/apache/cassandra/db/BufferExpiringCell.java +++ /dev/null @@ -1,187 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.apache.cassandra.db; - -import java.nio.ByteBuffer; -import java.security.MessageDigest; - -import org.apache.cassandra.config.CFMetaData; -import org.apache.cassandra.db.composites.CellName; -import org.apache.cassandra.db.composites.CellNameType; -import org.apache.cassandra.serializers.MarshalException; -import org.apache.cassandra.utils.FBUtilities; -import org.apache.cassandra.utils.concurrent.OpOrder; -import org.apache.cassandra.utils.memory.AbstractAllocator; -import org.apache.cassandra.utils.memory.MemtableAllocator; - -public class BufferExpiringCell extends BufferCell implements ExpiringCell -{ - private final int localExpirationTime; - private final int timeToLive; - - public BufferExpiringCell(CellName name, ByteBuffer value, long timestamp, int timeToLive) - { - this(name, value, timestamp, timeToLive, (int) (System.currentTimeMillis() / 1000) + timeToLive); - } - - public BufferExpiringCell(CellName name, ByteBuffer value, long timestamp, int timeToLive, int localExpirationTime) - { - super(name, value, timestamp); - assert timeToLive > 0 : timeToLive; - assert localExpirationTime > 0 : localExpirationTime; - this.timeToLive = timeToLive; - this.localExpirationTime = localExpirationTime; - } - - public int getTimeToLive() - { - return timeToLive; - } - - @Override - public Cell withUpdatedName(CellName newName) - { - return new BufferExpiringCell(newName, value(), timestamp(), timeToLive, localExpirationTime); - } - - @Override - public Cell withUpdatedTimestamp(long newTimestamp) - { - return new BufferExpiringCell(name(), value(), newTimestamp, timeToLive, localExpirationTime); - } - - @Override - public int cellDataSize() - { - return super.cellDataSize() + TypeSizes.NATIVE.sizeof(localExpirationTime) + TypeSizes.NATIVE.sizeof(timeToLive); - } - - @Override - public int serializedSize(CellNameType type, TypeSizes typeSizes) - { - /* - * An expired column adds to a Cell : - * 4 bytes for the localExpirationTime - * + 4 bytes for the timeToLive - */ - return super.serializedSize(type, typeSizes) + typeSizes.sizeof(localExpirationTime) + typeSizes.sizeof(timeToLive); - } - - @Override - public void updateDigest(MessageDigest digest) - { - super.updateDigest(digest); - FBUtilities.updateWithInt(digest, timeToLive); - } - - @Override - public int getLocalDeletionTime() - { - return localExpirationTime; - } - - @Override - public ExpiringCell localCopy(CFMetaData metadata, AbstractAllocator allocator) - { - return new BufferExpiringCell(name.copy(metadata, allocator), allocator.clone(value), timestamp, timeToLive, localExpirationTime); - } - - @Override - public ExpiringCell localCopy(CFMetaData metadata, MemtableAllocator allocator, OpOrder.Group opGroup) - { - return allocator.clone(this, metadata, opGroup); - } - - @Override - public String getString(CellNameType comparator) - { - return String.format("%s!%d", super.getString(comparator), timeToLive); - } - - @Override - public boolean isLive() - { - return isLive(System.currentTimeMillis()); - } - - @Override - public boolean isLive(long now) - { - return (int) (now / 1000) < getLocalDeletionTime(); - } - - @Override - public int serializationFlags() - { - return ColumnSerializer.EXPIRATION_MASK; - } - - @Override - public void validateFields(CFMetaData metadata) throws MarshalException - { - super.validateFields(metadata); - - if (timeToLive <= 0) - throw new MarshalException("A column TTL should be > 0, but was " + timeToLive); - if (localExpirationTime < 0) - throw new MarshalException("The local expiration time should not be negative but was " + localExpirationTime); - } - - @Override - public Cell reconcile(Cell cell) - { - long ts1 = timestamp(), ts2 = cell.timestamp(); - if (ts1 != ts2) - return ts1 < ts2 ? cell : this; - // we should prefer tombstones - if (cell instanceof DeletedCell) - return cell; - int c = value().compareTo(cell.value()); - if (c != 0) - return c < 0 ? cell : this; - // If we have same timestamp and value, prefer the longest ttl - if (cell instanceof ExpiringCell) - { - int let1 = localExpirationTime, let2 = cell.getLocalDeletionTime(); - if (let1 < let2) - return cell; - } - return this; - } - - @Override - public boolean equals(Cell cell) - { - if (!super.equals(cell)) - return false; - ExpiringCell that = (ExpiringCell) cell; - return getLocalDeletionTime() == that.getLocalDeletionTime() && getTimeToLive() == that.getTimeToLive(); - } - - /** @return Either a DeletedCell, or an ExpiringCell. */ - public static Cell create(CellName name, ByteBuffer value, long timestamp, int timeToLive, int localExpirationTime, int expireBefore, ColumnSerializer.Flag flag) - { - if (localExpirationTime >= expireBefore || flag == ColumnSerializer.Flag.PRESERVE_SIZE) - return new BufferExpiringCell(name, value, timestamp, timeToLive, localExpirationTime); - // The column is now expired, we can safely return a simple tombstone. Note that - // as long as the expiring column and the tombstone put together live longer than GC grace seconds, - // we'll fulfil our responsibility to repair. See discussion at - // http://cassandra-user-incubator-apache-org.3065146.n2.nabble.com/repair-compaction-and-tombstone-rows-td7583481.html - return new BufferDeletedCell(name, localExpirationTime - timeToLive, timestamp); - } -} diff --git a/src/java/org/apache/cassandra/db/CBuilder.java b/src/java/org/apache/cassandra/db/CBuilder.java new file mode 100644 index 0000000000..56cabf1743 --- /dev/null +++ b/src/java/org/apache/cassandra/db/CBuilder.java @@ -0,0 +1,231 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.cassandra.db; + +import java.nio.ByteBuffer; +import java.util.ArrayList; +import java.util.Arrays; +import java.util.List; + +import org.apache.cassandra.db.marshal.AbstractType; + +/** + * Allows to build ClusteringPrefixes, either Clustering or Slice.Bound. + */ +public abstract class CBuilder +{ + public static CBuilder STATIC_BUILDER = new CBuilder() + { + public int count() + { + return 0; + } + + public int remainingCount() + { + return 0; + } + + public ClusteringComparator comparator() + { + throw new UnsupportedOperationException(); + } + + public CBuilder add(ByteBuffer value) + { + throw new UnsupportedOperationException(); + } + + public CBuilder add(Object value) + { + throw new UnsupportedOperationException(); + } + + public Clustering build() + { + return Clustering.STATIC_CLUSTERING; + } + + public Slice.Bound buildBound(boolean isStart, boolean isInclusive) + { + throw new UnsupportedOperationException(); + } + + public Slice buildSlice() + { + throw new UnsupportedOperationException(); + } + + public Clustering buildWith(ByteBuffer value) + { + throw new UnsupportedOperationException(); + } + + public Clustering buildWith(List newValues) + { + throw new UnsupportedOperationException(); + } + + public Slice.Bound buildBoundWith(ByteBuffer value, boolean isStart, boolean isInclusive) + { + throw new UnsupportedOperationException(); + } + + public Slice.Bound buildBoundWith(List newValues, boolean isStart, boolean isInclusive) + { + throw new UnsupportedOperationException(); + } + }; + + public static CBuilder create(ClusteringComparator comparator) + { + return new ArrayBackedBuilder(comparator); + } + + public abstract int count(); + public abstract int remainingCount(); + public abstract ClusteringComparator comparator(); + public abstract CBuilder add(ByteBuffer value); + public abstract CBuilder add(Object value); + public abstract Clustering build(); + public abstract Slice.Bound buildBound(boolean isStart, boolean isInclusive); + public abstract Slice buildSlice(); + public abstract Clustering buildWith(ByteBuffer value); + public abstract Clustering buildWith(List newValues); + public abstract Slice.Bound buildBoundWith(ByteBuffer value, boolean isStart, boolean isInclusive); + public abstract Slice.Bound buildBoundWith(List newValues, boolean isStart, boolean isInclusive); + + private static class ArrayBackedBuilder extends CBuilder + { + private final ClusteringComparator type; + private final ByteBuffer[] values; + private int size; + private boolean built; + + public ArrayBackedBuilder(ClusteringComparator type) + { + this.type = type; + this.values = new ByteBuffer[type.size()]; + } + + public int count() + { + return size; + } + + public int remainingCount() + { + return values.length - size; + } + + public ClusteringComparator comparator() + { + return type; + } + + public CBuilder add(ByteBuffer value) + { + if (isDone()) + throw new IllegalStateException(); + values[size++] = value; + return this; + } + + public CBuilder add(Object value) + { + return add(((AbstractType)type.subtype(size)).decompose(value)); + } + + private boolean isDone() + { + return remainingCount() == 0 || built; + } + + public Clustering build() + { + // We don't allow to add more element to a builder that has been built so + // that we don't have to copy values. + built = true; + + // Currently, only dense table can leave some clustering column out (see #7990) + return size == 0 ? Clustering.EMPTY : new SimpleClustering(values); + } + + public Slice.Bound buildBound(boolean isStart, boolean isInclusive) + { + // We don't allow to add more element to a builder that has been built so + // that we don't have to copy values (even though we have to do it in most cases). + built = true; + + if (size == 0) + return isStart ? Slice.Bound.BOTTOM : Slice.Bound.TOP; + + return Slice.Bound.create(Slice.Bound.boundKind(isStart, isInclusive), + size == values.length ? values : Arrays.copyOfRange(values, 0, size)); + } + + public Slice buildSlice() + { + // We don't allow to add more element to a builder that has been built so + // that we don't have to copy values. + built = true; + + if (size == 0) + return Slice.ALL; + + return Slice.make(buildBound(true, true), buildBound(false, true)); + } + + public Clustering buildWith(ByteBuffer value) + { + assert size+1 == type.size(); + + ByteBuffer[] newValues = Arrays.copyOf(values, size+1); + newValues[size] = value; + return new SimpleClustering(newValues); + } + + public Clustering buildWith(List newValues) + { + assert size + newValues.size() == type.size(); + ByteBuffer[] buffers = Arrays.copyOf(values, size + newValues.size()); + int newSize = size; + for (ByteBuffer value : newValues) + buffers[newSize++] = value; + + return new SimpleClustering(buffers); + } + + public Slice.Bound buildBoundWith(ByteBuffer value, boolean isStart, boolean isInclusive) + { + ByteBuffer[] newValues = Arrays.copyOf(values, size+1); + newValues[size] = value; + return Slice.Bound.create(Slice.Bound.boundKind(isStart, isInclusive), newValues); + } + + public Slice.Bound buildBoundWith(List newValues, boolean isStart, boolean isInclusive) + { + ByteBuffer[] buffers = Arrays.copyOf(values, size + newValues.size()); + int newSize = size; + for (ByteBuffer value : newValues) + buffers[newSize++] = value; + + return Slice.Bound.create(Slice.Bound.boundKind(isStart, isInclusive), buffers); + } + } +} diff --git a/src/java/org/apache/cassandra/db/CFRowAdder.java b/src/java/org/apache/cassandra/db/CFRowAdder.java deleted file mode 100644 index 6fab8d51cc..0000000000 --- a/src/java/org/apache/cassandra/db/CFRowAdder.java +++ /dev/null @@ -1,121 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.apache.cassandra.db; - -import java.nio.ByteBuffer; - -import org.apache.cassandra.cql3.ColumnIdentifier; -import org.apache.cassandra.config.ColumnDefinition; -import org.apache.cassandra.db.composites.CellName; -import org.apache.cassandra.db.composites.Composite; -import org.apache.cassandra.db.marshal.AbstractType; -import org.apache.cassandra.db.marshal.CollectionType; -import org.apache.cassandra.db.marshal.ListType; -import org.apache.cassandra.db.marshal.MapType; -import org.apache.cassandra.utils.ByteBufferUtil; -import org.apache.cassandra.utils.UUIDGen; - -/** - * Convenience object to populate a given CQL3 row in a ColumnFamily object. - * - * This is meant for when performance is not of the utmost importance. When - * performance matters, it might be worth allocating such builder. - */ -public class CFRowAdder -{ - public final ColumnFamily cf; - public final Composite prefix; - public final long timestamp; - public final int ttl; - private final int ldt; - - public CFRowAdder(ColumnFamily cf, Composite prefix, long timestamp) - { - this(cf, prefix, timestamp, 0); - } - - public CFRowAdder(ColumnFamily cf, Composite prefix, long timestamp, int ttl) - { - this.cf = cf; - this.prefix = prefix; - this.timestamp = timestamp; - this.ttl = ttl; - this.ldt = (int) (System.currentTimeMillis() / 1000); - - // If a CQL3 table, add the row marker - if (cf.metadata().isCQL3Table() && !prefix.isStatic()) - cf.addColumn(new BufferCell(cf.getComparator().rowMarker(prefix), ByteBufferUtil.EMPTY_BYTE_BUFFER, timestamp)); - } - - public CFRowAdder add(String cql3ColumnName, Object value) - { - ColumnDefinition def = getDefinition(cql3ColumnName); - return add(cf.getComparator().create(prefix, def), def, value); - } - - public CFRowAdder resetCollection(String cql3ColumnName) - { - ColumnDefinition def = getDefinition(cql3ColumnName); - assert def.type.isCollection() && def.type.isMultiCell(); - Composite name = cf.getComparator().create(prefix, def); - cf.addAtom(new RangeTombstone(name.start(), name.end(), timestamp - 1, ldt)); - return this; - } - - public CFRowAdder addMapEntry(String cql3ColumnName, Object key, Object value) - { - ColumnDefinition def = getDefinition(cql3ColumnName); - assert def.type instanceof MapType; - MapType mt = (MapType)def.type; - CellName name = cf.getComparator().create(prefix, def, mt.getKeysType().decompose(key)); - return add(name, def, value); - } - - public CFRowAdder addListEntry(String cql3ColumnName, Object value) - { - ColumnDefinition def = getDefinition(cql3ColumnName); - assert def.type instanceof ListType; - CellName name = cf.getComparator().create(prefix, def, ByteBuffer.wrap(UUIDGen.getTimeUUIDBytes())); - return add(name, def, value); - } - - private ColumnDefinition getDefinition(String name) - { - return cf.metadata().getColumnDefinition(new ColumnIdentifier(name, false)); - } - - private CFRowAdder add(CellName name, ColumnDefinition def, Object value) - { - if (value == null) - { - cf.addColumn(new BufferDeletedCell(name, ldt, timestamp)); - } - else - { - AbstractType valueType = def.type.isCollection() - ? ((CollectionType) def.type).valueComparator() - : def.type; - ByteBuffer valueBytes = value instanceof ByteBuffer ? (ByteBuffer)value : valueType.decompose(value); - if (ttl == 0) - cf.addColumn(new BufferCell(name, valueBytes, timestamp)); - else - cf.addColumn(new BufferExpiringCell(name, valueBytes, timestamp, ttl)); - } - return this; - } -} diff --git a/src/java/org/apache/cassandra/db/Cell.java b/src/java/org/apache/cassandra/db/Cell.java deleted file mode 100644 index 7c3926ac9d..0000000000 --- a/src/java/org/apache/cassandra/db/Cell.java +++ /dev/null @@ -1,69 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.apache.cassandra.db; - -import java.nio.ByteBuffer; - -import org.apache.cassandra.config.CFMetaData; -import org.apache.cassandra.db.composites.CellName; -import org.apache.cassandra.db.composites.CellNameType; -import org.apache.cassandra.utils.concurrent.OpOrder; -import org.apache.cassandra.utils.memory.AbstractAllocator; -import org.apache.cassandra.utils.FBUtilities; -import org.apache.cassandra.utils.memory.MemtableAllocator; - -/** - * Cell is immutable, which prevents all kinds of confusion in a multithreaded environment. - */ -public interface Cell extends OnDiskAtom -{ - public static final int MAX_NAME_LENGTH = FBUtilities.MAX_UNSIGNED_SHORT; - - public Cell withUpdatedName(CellName newName); - - public Cell withUpdatedTimestamp(long newTimestamp); - - @Override - public CellName name(); - - public ByteBuffer value(); - - public boolean isLive(); - - public boolean isLive(long now); - - public int cellDataSize(); - - // returns the size of the Cell and all references on the heap, excluding any costs associated with byte arrays - // that would be allocated by a localCopy, as these will be accounted for by the allocator - public long unsharedHeapSizeExcludingData(); - - public int serializedSize(CellNameType type, TypeSizes typeSizes); - - public int serializationFlags(); - - public Cell diff(Cell cell); - - public Cell reconcile(Cell cell); - - public Cell localCopy(CFMetaData metadata, AbstractAllocator allocator); - - public Cell localCopy(CFMetaData metaData, MemtableAllocator allocator, OpOrder.Group opGroup); - - public String getString(CellNameType comparator); -} diff --git a/src/java/org/apache/cassandra/service/IReadCommand.java b/src/java/org/apache/cassandra/db/Clusterable.java similarity index 76% rename from src/java/org/apache/cassandra/service/IReadCommand.java rename to src/java/org/apache/cassandra/db/Clusterable.java index c6a129e268..62ab9dc67c 100644 --- a/src/java/org/apache/cassandra/service/IReadCommand.java +++ b/src/java/org/apache/cassandra/db/Clusterable.java @@ -15,10 +15,13 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -package org.apache.cassandra.service; +package org.apache.cassandra.db; -public interface IReadCommand +/** + * Common class for objects that are identified by a clustering prefix, and can be thus sorted by a + * {@link ClusteringComparator}. + */ +public interface Clusterable { - public String getKeyspace(); - public long getTimeout(); + public ClusteringPrefix clustering(); } diff --git a/src/java/org/apache/cassandra/db/Clustering.java b/src/java/org/apache/cassandra/db/Clustering.java new file mode 100644 index 0000000000..5ac967197d --- /dev/null +++ b/src/java/org/apache/cassandra/db/Clustering.java @@ -0,0 +1,171 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.cassandra.db; + +import java.io.DataInput; +import java.io.IOException; +import java.nio.ByteBuffer; +import java.util.*; + +import org.apache.cassandra.config.CFMetaData; +import org.apache.cassandra.config.ColumnDefinition; +import org.apache.cassandra.db.marshal.AbstractType; +import org.apache.cassandra.io.util.DataOutputPlus; + +/** + * The clustering column values for a row. + *

+ * A {@code Clustering} is a {@code ClusteringPrefix} that must always be "complete", i.e. have + * as many values as there is clustering columns in the table it is part of. It is the clustering + * prefix used by rows. + *

+ * Note however that while it's size must be equal to the table clustering size, a clustering can have + * {@code null} values, and this mostly for thrift backward compatibility (in practice, if a value is null, + * all of the following ones will be too because that's what thrift allows, but it's never assumed by the + * code so we could start generally allowing nulls for clustering columns if we wanted to). + */ +public abstract class Clustering extends AbstractClusteringPrefix +{ + public static final Serializer serializer = new Serializer(); + + /** + * The special cased clustering used by all static rows. It is a special case in the + * sense that it's always empty, no matter how many clustering columns the table has. + */ + public static final Clustering STATIC_CLUSTERING = new EmptyClustering() + { + @Override + public Kind kind() + { + return Kind.STATIC_CLUSTERING; + } + + @Override + public String toString(CFMetaData metadata) + { + return "STATIC"; + } + }; + + /** Empty clustering for tables having no clustering columns. */ + public static final Clustering EMPTY = new EmptyClustering(); + + public Kind kind() + { + return Kind.CLUSTERING; + } + + public Clustering takeAlias() + { + ByteBuffer[] values = new ByteBuffer[size()]; + for (int i = 0; i < size(); i++) + values[i] = get(i); + return new SimpleClustering(values); + } + + public String toString(CFMetaData metadata) + { + StringBuilder sb = new StringBuilder(); + for (int i = 0; i < size(); i++) + { + ColumnDefinition c = metadata.clusteringColumns().get(i); + sb.append(i == 0 ? "" : ", ").append(c.name).append("=").append(get(i) == null ? "null" : c.type.getString(get(i))); + } + return sb.toString(); + } + + public String toCQLString(CFMetaData metadata) + { + StringBuilder sb = new StringBuilder(); + for (int i = 0; i < size(); i++) + { + ColumnDefinition c = metadata.clusteringColumns().get(i); + sb.append(i == 0 ? "" : ", ").append(c.type.getString(get(i))); + } + return sb.toString(); + } + + private static class EmptyClustering extends Clustering + { + private static final ByteBuffer[] EMPTY_VALUES_ARRAY = new ByteBuffer[0]; + + public int size() + { + return 0; + } + + public ByteBuffer get(int i) + { + throw new UnsupportedOperationException(); + } + + public ByteBuffer[] getRawValues() + { + return EMPTY_VALUES_ARRAY; + } + + @Override + public Clustering takeAlias() + { + return this; + } + + @Override + public long unsharedHeapSize() + { + return 0; + } + + @Override + public String toString(CFMetaData metadata) + { + return "EMPTY"; + } + } + + /** + * Serializer for Clustering object. + *

+ * Because every clustering in a given table must have the same size (ant that size cannot actually change once the table + * has been defined), we don't record that size. + */ + public static class Serializer + { + public void serialize(Clustering clustering, DataOutputPlus out, int version, List> types) throws IOException + { + ClusteringPrefix.serializer.serializeValuesWithoutSize(clustering, out, version, types); + } + + public long serializedSize(Clustering clustering, int version, List> types, TypeSizes sizes) + { + return ClusteringPrefix.serializer.valuesWithoutSizeSerializedSize(clustering, version, types, sizes); + } + + public void deserialize(DataInput in, int version, List> types, Writer writer) throws IOException + { + ClusteringPrefix.serializer.deserializeValuesWithoutSize(in, types.size(), version, types, writer); + } + + public Clustering deserialize(DataInput in, int version, List> types) throws IOException + { + SimpleClustering.Builder builder = SimpleClustering.builder(types.size()); + deserialize(in, version, types, builder); + return builder.build(); + } + } +} diff --git a/src/java/org/apache/cassandra/db/ClusteringComparator.java b/src/java/org/apache/cassandra/db/ClusteringComparator.java new file mode 100644 index 0000000000..b0e8e5ca89 --- /dev/null +++ b/src/java/org/apache/cassandra/db/ClusteringComparator.java @@ -0,0 +1,291 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.cassandra.db; + +import java.nio.ByteBuffer; +import java.util.Arrays; +import java.util.Comparator; +import java.util.List; +import java.util.Objects; + +import com.google.common.base.Joiner; + +import org.apache.cassandra.db.marshal.AbstractType; +import org.apache.cassandra.utils.ByteBufferUtil; + +import static org.apache.cassandra.io.sstable.IndexHelper.IndexInfo; + +/** + * A comparator of clustering prefixes (or more generally of {@link Clusterable}}. + *

+ * This is essentially just a composite comparator that the clustering values of the provided + * clustering prefixes in lexicographical order, with each component being compared based on + * the type of the clustering column this is a value of. + */ +public class ClusteringComparator implements Comparator +{ + private final List> clusteringTypes; + private final boolean isByteOrderComparable; + + private final Comparator indexComparator; + private final Comparator indexReverseComparator; + private final Comparator reverseComparator; + + public ClusteringComparator(AbstractType... clusteringTypes) + { + this(Arrays.>asList(clusteringTypes)); + } + + public ClusteringComparator(List> clusteringTypes) + { + this.clusteringTypes = clusteringTypes; + this.isByteOrderComparable = isByteOrderComparable(clusteringTypes); + + this.indexComparator = new Comparator() + { + public int compare(IndexInfo o1, IndexInfo o2) + { + return ClusteringComparator.this.compare(o1.lastName, o2.lastName); + } + }; + this.indexReverseComparator = new Comparator() + { + public int compare(IndexInfo o1, IndexInfo o2) + { + return ClusteringComparator.this.compare(o1.firstName, o2.firstName); + } + }; + this.reverseComparator = new Comparator() + { + public int compare(Clusterable c1, Clusterable c2) + { + return ClusteringComparator.this.compare(c2, c1); + } + }; + } + + private static boolean isByteOrderComparable(Iterable> types) + { + boolean isByteOrderComparable = true; + for (AbstractType type : types) + isByteOrderComparable &= type.isByteOrderComparable(); + return isByteOrderComparable; + } + + /** + * The number of clustering columns for the table this is the comparator of. + */ + public int size() + { + return clusteringTypes.size(); + } + + /** + * The "subtypes" of this clustering comparator, that is the types of the clustering + * columns for the table this is a comparator of. + */ + public List> subtypes() + { + return clusteringTypes; + } + + /** + * Returns the type of the ith clustering column of the table. + */ + public AbstractType subtype(int i) + { + return clusteringTypes.get(i); + } + + /** + * Creates a row clustering based on the clustering values. + *

+ * Every argument can either be a {@code ByteBuffer}, in which case it is used as-is, or a object + * corresponding to the type of the corresponding clustering column, in which case it will be + * converted to a byte buffer using the column type. + * + * @param values the values to use for the created clustering. There should be exactly {@code size()} + * values which must be either byte buffers or of the type the column expect. + * + * @return the newly created clustering. + */ + public Clustering make(Object... values) + { + if (values.length != size()) + throw new IllegalArgumentException(String.format("Invalid number of components, expecting %d but got %d", size(), values.length)); + + CBuilder builder = CBuilder.create(this); + for (int i = 0; i < values.length; i++) + { + Object val = values[i]; + if (val instanceof ByteBuffer) + builder.add((ByteBuffer)val); + else + builder.add(val); + } + return builder.build(); + } + + public int compare(Clusterable c1, Clusterable c2) + { + return compare(c1.clustering(), c2.clustering()); + } + + public int compare(ClusteringPrefix c1, ClusteringPrefix c2) + { + int s1 = c1.size(); + int s2 = c2.size(); + int minSize = Math.min(s1, s2); + + for (int i = 0; i < minSize; i++) + { + int cmp = compareComponent(i, c1.get(i), c2.get(i)); + if (cmp != 0) + return cmp; + } + + if (s1 == s2) + return ClusteringPrefix.Kind.compare(c1.kind(), c2.kind()); + + return s1 < s2 ? c1.kind().prefixComparisonResult : -c2.kind().prefixComparisonResult; + } + + public int compare(Clustering c1, Clustering c2) + { + for (int i = 0; i < size(); i++) + { + int cmp = compareComponent(i, c1.get(i), c2.get(i)); + if (cmp != 0) + return cmp; + } + return 0; + } + + public int compareComponent(int i, ByteBuffer v1, ByteBuffer v2) + { + if (v1 == null) + return v1 == null ? 0 : -1; + if (v2 == null) + return 1; + + return isByteOrderComparable + ? ByteBufferUtil.compareUnsigned(v1, v2) + : clusteringTypes.get(i).compare(v1, v2); + } + + /** + * Returns whether this clustering comparator is compatible with the provided one, + * that is if the provided one can be safely replaced by this new one. + * + * @param previous the previous comparator that we want to replace and test + * compatibility with. + * + * @return whether {@code previous} can be safely replaced by this comparator. + */ + public boolean isCompatibleWith(ClusteringComparator previous) + { + if (this == previous) + return true; + + // Extending with new components is fine, shrinking is not + if (size() < previous.size()) + return false; + + for (int i = 0; i < previous.size(); i++) + { + AbstractType tprev = previous.subtype(i); + AbstractType tnew = subtype(i); + if (!tnew.isCompatibleWith(tprev)) + return false; + } + return true; + } + + /** + * Validates the provided prefix for corrupted data. + * + * @param clustering the clustering prefix to validate. + * + * @throws MarshalException if {@code clustering} contains some invalid data. + */ + public void validate(ClusteringPrefix clustering) + { + for (int i = 0; i < clustering.size(); i++) + { + ByteBuffer value = clustering.get(i); + if (value != null) + subtype(i).validate(value); + } + } + + public Comparator indexComparator(boolean reversed) + { + return reversed ? indexReverseComparator : indexComparator; + } + + public Comparator reversed() + { + return reverseComparator; + } + + /** + * Whether the two provided clustering prefix are on the same clustering values. + * + * @param c1 the first prefix. + * @param c2 the second prefix. + * @return whether {@code c1} and {@code c2} have the same clustering values (but not necessarily + * the same "kind") or not. + */ + public boolean isOnSameClustering(ClusteringPrefix c1, ClusteringPrefix c2) + { + if (c1.size() != c2.size()) + return false; + + for (int i = 0; i < c1.size(); i++) + { + if (compareComponent(i, c1.get(i), c2.get(i)) != 0) + return false; + } + return true; + } + + @Override + public String toString() + { + return String.format("comparator(%s)", Joiner.on(", ").join(clusteringTypes)); + } + + @Override + public boolean equals(Object o) + { + if (this == o) + return true; + + if (!(o instanceof ClusteringComparator)) + return false; + + ClusteringComparator that = (ClusteringComparator)o; + return this.clusteringTypes.equals(that.clusteringTypes); + } + + @Override + public int hashCode() + { + return Objects.hashCode(clusteringTypes); + } +} diff --git a/src/java/org/apache/cassandra/db/ClusteringPrefix.java b/src/java/org/apache/cassandra/db/ClusteringPrefix.java new file mode 100644 index 0000000000..73cedb8a8d --- /dev/null +++ b/src/java/org/apache/cassandra/db/ClusteringPrefix.java @@ -0,0 +1,513 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.cassandra.db; + +import java.io.DataInput; +import java.io.IOException; +import java.nio.ByteBuffer; +import java.security.MessageDigest; +import java.util.*; + +import org.apache.cassandra.cache.IMeasurableMemory; +import org.apache.cassandra.config.CFMetaData; +import org.apache.cassandra.db.rows.*; +import org.apache.cassandra.db.marshal.AbstractType; +import org.apache.cassandra.io.util.DataOutputPlus; +import org.apache.cassandra.utils.ByteBufferUtil; + +/** + * A clustering prefix is basically the unit of what a {@link ClusteringComparator} can compare. + *

+ * It holds values for the clustering columns of a table (potentially only a prefix of all of them) and it has + * a "kind" that allows us to implement slices with inclusive and exclusive bounds. + *

+ * In practice, {@code ClusteringPrefix} is just the common parts to its 2 main subtype: {@link Clustering} and + * {@link Slice.Bound}, where: + * 1) {@code Clustering} represents the clustering values for a row, i.e. the values for it's clustering columns. + * 2) {@code Slice.Bound} represents a bound (start or end) of a slice (of rows). + * See those classes for more details. + */ +public interface ClusteringPrefix extends Aliasable, IMeasurableMemory, Clusterable +{ + public static final Serializer serializer = new Serializer(); + + /** + * The kind of clustering prefix this actually is. + * + * The kind {@code STATIC_CLUSTERING} is only implemented by {@link Clustering.STATIC_CLUSTERING} and {@code CLUSTERING} is + * implemented by the {@link Clustering} class. The rest is used by {@link Slice.Bound} and {@link RangeTombstone.Bound}. + */ + public enum Kind + { + // WARNING: the ordering of that enum matters because we use ordinal() in the serialization + + EXCL_END_BOUND(0, -1), + INCL_START_BOUND(1, -1), + EXCL_END_INCL_START_BOUNDARY(1, -1), + STATIC_CLUSTERING(2, -1), + CLUSTERING(3, 0), + INCL_END_EXCL_START_BOUNDARY(4, -1), + INCL_END_BOUND(4, 1), + EXCL_START_BOUND(5, 1); + + private final int comparison; + + // If clusterable c1 has this Kind and is a strict prefix of clusterable c2, then this + // is the result of compare(c1, c2). Basically, this is the same as comparing the kind of c1 to + // CLUSTERING. + public final int prefixComparisonResult; + + private Kind(int comparison, int prefixComparisonResult) + { + this.comparison = comparison; + this.prefixComparisonResult = prefixComparisonResult; + } + + /** + * Compares the 2 provided kind. + *

+ * Note: this should be used instead of {@link #compareTo} when comparing clustering prefixes. We do + * not override that latter method because it is final for an enum. + */ + public static int compare(Kind k1, Kind k2) + { + return Integer.compare(k1.comparison, k2.comparison); + } + + /** + * Returns the inverse of the current kind. + *

+ * This invert both start into end (and vice-versa) and inclusive into exclusive (and vice-versa). + * + * @return the invert of this kind. For instance, if this kind is an exlusive start, this return + * an inclusive end. + */ + public Kind invert() + { + switch (this) + { + case EXCL_START_BOUND: return INCL_END_BOUND; + case INCL_START_BOUND: return EXCL_END_BOUND; + case EXCL_END_BOUND: return INCL_START_BOUND; + case INCL_END_BOUND: return EXCL_START_BOUND; + case EXCL_END_INCL_START_BOUNDARY: return INCL_END_EXCL_START_BOUNDARY; + case INCL_END_EXCL_START_BOUNDARY: return EXCL_END_INCL_START_BOUNDARY; + default: return this; + } + } + + public boolean isBound() + { + switch (this) + { + case INCL_START_BOUND: + case INCL_END_BOUND: + case EXCL_START_BOUND: + case EXCL_END_BOUND: + return true; + } + return false; + } + + public boolean isBoundary() + { + switch (this) + { + case INCL_END_EXCL_START_BOUNDARY: + case EXCL_END_INCL_START_BOUNDARY: + return true; + } + return false; + } + + public boolean isStart() + { + switch (this) + { + case INCL_START_BOUND: + case EXCL_END_INCL_START_BOUNDARY: + case INCL_END_EXCL_START_BOUNDARY: + case EXCL_START_BOUND: + return true; + default: + return false; + } + } + + public boolean isEnd() + { + switch (this) + { + case INCL_END_BOUND: + case EXCL_END_INCL_START_BOUNDARY: + case INCL_END_EXCL_START_BOUNDARY: + case EXCL_END_BOUND: + return true; + default: + return false; + } + } + + public boolean isOpen(boolean reversed) + { + return reversed ? isEnd() : isStart(); + } + + public boolean isClose(boolean reversed) + { + return reversed ? isStart() : isEnd(); + } + + public Kind closeBoundOfBoundary(boolean reversed) + { + assert isBoundary(); + return reversed + ? (this == INCL_END_EXCL_START_BOUNDARY ? EXCL_START_BOUND : INCL_START_BOUND) + : (this == INCL_END_EXCL_START_BOUNDARY ? INCL_END_BOUND : EXCL_END_BOUND); + } + + public Kind openBoundOfBoundary(boolean reversed) + { + assert isBoundary(); + return reversed + ? (this == INCL_END_EXCL_START_BOUNDARY ? INCL_END_BOUND : EXCL_END_BOUND) + : (this == INCL_END_EXCL_START_BOUNDARY ? EXCL_START_BOUND : INCL_START_BOUND); + } + } + + public Kind kind(); + + /** + * The number of values in this prefix. + * + * There can't be more values that the this is a prefix of has of clustering columns. + * + * @return the number of values in this prefix. + */ + public int size(); + + /** + * Retrieves the ith value of this prefix. + * + * @param i the index of the value to retrieve. Must be such that {@code 0 <= i < size()}. + * + * @return the ith value of this prefix. Note that a value can be {@code null}. + */ + public ByteBuffer get(int i); + + public void digest(MessageDigest digest); + + // Used to verify if batches goes over a given size + public int dataSize(); + + public String toString(CFMetaData metadata); + + public void writeTo(Writer writer); + + /** + * The values of this prefix as an array. + *

+ * Please note that this may or may not require an array creation. So 1) you should *not* + * modify the returned array and 2) it's more efficient to use {@link #size()} and + * {@link #get} unless you actually need an array. + * + * @return the values for this prefix as an array. + */ + public ByteBuffer[] getRawValues(); + + /** + * Interface for writing a clustering prefix. + *

+ * Each value for the prefix should simply be written in order. + */ + public interface Writer + { + /** + * Write the next value to the writer. + * + * @param value the value to write. + */ + public void writeClusteringValue(ByteBuffer value); + } + + public static class Serializer + { + public void serialize(ClusteringPrefix clustering, DataOutputPlus out, int version, List> types) throws IOException + { + // We shouldn't serialize static clusterings + assert clustering.kind() != Kind.STATIC_CLUSTERING; + if (clustering.kind() == Kind.CLUSTERING) + { + out.writeByte(clustering.kind().ordinal()); + Clustering.serializer.serialize((Clustering)clustering, out, version, types); + } + else + { + Slice.Bound.serializer.serialize((Slice.Bound)clustering, out, version, types); + } + } + + public ClusteringPrefix deserialize(DataInput in, int version, List> types) throws IOException + { + Kind kind = Kind.values()[in.readByte()]; + // We shouldn't serialize static clusterings + assert kind != Kind.STATIC_CLUSTERING; + if (kind == Kind.CLUSTERING) + return Clustering.serializer.deserialize(in, version, types); + else + return Slice.Bound.serializer.deserializeValues(in, kind, version, types); + } + + public long serializedSize(ClusteringPrefix clustering, int version, List> types, TypeSizes sizes) + { + // We shouldn't serialize static clusterings + assert clustering.kind() != Kind.STATIC_CLUSTERING; + if (clustering.kind() == Kind.CLUSTERING) + return 1 + Clustering.serializer.serializedSize((Clustering)clustering, version, types, sizes); + else + return Slice.Bound.serializer.serializedSize((Slice.Bound)clustering, version, types, sizes); + } + + void serializeValuesWithoutSize(ClusteringPrefix clustering, DataOutputPlus out, int version, List> types) throws IOException + { + if (clustering.size() == 0) + return; + + writeHeader(clustering, out); + for (int i = 0; i < clustering.size(); i++) + { + ByteBuffer v = clustering.get(i); + if (v == null || !v.hasRemaining()) + continue; // handled in the header + + types.get(i).writeValue(v, out); + } + } + + long valuesWithoutSizeSerializedSize(ClusteringPrefix clustering, int version, List> types, TypeSizes sizes) + { + if (clustering.size() == 0) + return 0; + + long size = headerBytesCount(clustering.size()); + for (int i = 0; i < clustering.size(); i++) + { + ByteBuffer v = clustering.get(i); + if (v == null || !v.hasRemaining()) + continue; // handled in the header + + size += types.get(i).writtenLength(v, sizes); + } + return size; + } + + void deserializeValuesWithoutSize(DataInput in, int size, int version, List> types, ClusteringPrefix.Writer writer) throws IOException + { + if (size == 0) + return; + + int[] header = readHeader(size, in); + for (int i = 0; i < size; i++) + { + if (isNull(header, i)) + writer.writeClusteringValue(null); + else if (isEmpty(header, i)) + writer.writeClusteringValue(ByteBufferUtil.EMPTY_BYTE_BUFFER); + else + writer.writeClusteringValue(types.get(i).readValue(in)); + } + } + + private int headerBytesCount(int size) + { + // For each component, we store 2 bit to know if the component is empty or null (or neither). + // We thus handle 4 component per byte + return size / 4 + (size % 4 == 0 ? 0 : 1); + } + + /** + * Whatever the type of a given clustering column is, its value can always be either empty or null. So we at least need to distinguish those + * 2 values, and because we want to be able to store fixed width values without appending their (fixed) size first, we need a way to encode + * empty values too. So for that, every clustering prefix includes a "header" that contains 2 bits per element in the prefix. For each element, + * those 2 bits encode whether the element is null, empty, or none of those. + */ + private void writeHeader(ClusteringPrefix clustering, DataOutputPlus out) throws IOException + { + int nbBytes = headerBytesCount(clustering.size()); + for (int i = 0; i < nbBytes; i++) + { + int b = 0; + for (int j = 0; j < 4; j++) + { + int c = i * 4 + j; + if (c >= clustering.size()) + break; + + ByteBuffer v = clustering.get(c); + if (v == null) + b |= (1 << (j * 2) + 1); + else if (!v.hasRemaining()) + b |= (1 << (j * 2)); + } + out.writeByte((byte)b); + } + } + + private int[] readHeader(int size, DataInput in) throws IOException + { + int nbBytes = headerBytesCount(size); + int[] header = new int[nbBytes]; + for (int i = 0; i < nbBytes; i++) + header[i] = in.readUnsignedByte(); + return header; + } + + private boolean isNull(int[] header, int i) + { + int b = header[i / 4]; + int mask = 1 << ((i % 4) * 2) + 1; + return (b & mask) != 0; + } + + private boolean isEmpty(int[] header, int i) + { + int b = header[i / 4]; + int mask = 1 << ((i % 4) * 2); + return (b & mask) != 0; + } + } + + /** + * Helper class that makes the deserialization of clustering prefixes faster. + *

+ * The main reason for this is that when we deserialize rows from sstables, there is many cases where we have + * a bunch of rows to skip at the beginning of an index block because those rows are before the requested slice. + * This class make sure we can answer the question "is the next row on disk before the requested slice" with as + * little work as possible. It does that by providing a comparison method that deserialize only what is needed + * to decide of the comparison. + */ + public static class Deserializer + { + private final ClusteringComparator comparator; + private final DataInput in; + private final SerializationHeader serializationHeader; + + private boolean nextIsRow; + private int[] nextHeader; + + private int nextSize; + private ClusteringPrefix.Kind nextKind; + private int deserializedSize; + private final ByteBuffer[] nextValues; + + public Deserializer(ClusteringComparator comparator, DataInput in, SerializationHeader header) + { + this.comparator = comparator; + this.in = in; + this.serializationHeader = header; + this.nextValues = new ByteBuffer[comparator.size()]; + } + + public void prepare(int flags) throws IOException + { + assert !UnfilteredSerializer.isStatic(flags) : "Flags = " + flags; + this.nextIsRow = UnfilteredSerializer.kind(flags) == Unfiltered.Kind.ROW; + this.nextKind = nextIsRow ? Kind.CLUSTERING : ClusteringPrefix.Kind.values()[in.readByte()]; + this.nextSize = nextIsRow ? comparator.size() : in.readUnsignedShort(); + this.nextHeader = serializer.readHeader(nextSize, in); + this.deserializedSize = 0; + } + + public int compareNextTo(Slice.Bound bound) throws IOException + { + if (bound == Slice.Bound.TOP) + return -1; + + for (int i = 0; i < bound.size(); i++) + { + if (!hasComponent(i)) + return nextKind.prefixComparisonResult; + + int cmp = comparator.compareComponent(i, nextValues[i], bound.get(i)); + if (cmp != 0) + return cmp; + } + + if (bound.size() == nextSize) + return nextKind.compareTo(bound.kind()); + + // We know that we'll have exited already if nextSize < bound.size + return -bound.kind().prefixComparisonResult; + } + + private boolean hasComponent(int i) throws IOException + { + if (i >= nextSize) + return false; + + while (deserializedSize <= i) + deserializeOne(); + + return true; + } + + private boolean deserializeOne() throws IOException + { + if (deserializedSize == nextSize) + return false; + + int i = deserializedSize++; + nextValues[i] = serializer.isNull(nextHeader, i) + ? null + : (serializer.isEmpty(nextHeader, i) ? ByteBufferUtil.EMPTY_BYTE_BUFFER : serializationHeader.clusteringTypes().get(i).readValue(in)); + return true; + } + + private void deserializeAll() throws IOException + { + while (deserializeOne()) + continue; + } + + public RangeTombstone.Bound.Kind deserializeNextBound(RangeTombstone.Bound.Writer writer) throws IOException + { + assert !nextIsRow; + deserializeAll(); + for (int i = 0; i < nextSize; i++) + writer.writeClusteringValue(nextValues[i]); + writer.writeBoundKind(nextKind); + return nextKind; + } + + public void deserializeNextClustering(Clustering.Writer writer) throws IOException + { + assert nextIsRow && nextSize == nextValues.length; + deserializeAll(); + for (int i = 0; i < nextSize; i++) + writer.writeClusteringValue(nextValues[i]); + } + + public ClusteringPrefix.Kind skipNext() throws IOException + { + for (int i = deserializedSize; i < nextSize; i++) + if (!serializer.isNull(nextHeader, i) && !serializer.isEmpty(nextHeader, i)) + serializationHeader.clusteringTypes().get(i).skipValue(in); + return nextKind; + } + } +} diff --git a/src/java/org/apache/cassandra/db/CollationController.java b/src/java/org/apache/cassandra/db/CollationController.java deleted file mode 100644 index 7f6d439b23..0000000000 --- a/src/java/org/apache/cassandra/db/CollationController.java +++ /dev/null @@ -1,334 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.apache.cassandra.db; - -import java.io.Closeable; -import java.util.ArrayList; -import java.util.Collections; -import java.util.Iterator; -import java.util.List; -import java.util.TreeSet; - -import com.google.common.base.Function; -import com.google.common.collect.Iterables; -import com.google.common.collect.Iterators; - -import org.apache.cassandra.concurrent.Stage; -import org.apache.cassandra.concurrent.StageManager; -import org.apache.cassandra.db.columniterator.OnDiskAtomIterator; -import org.apache.cassandra.db.composites.CellName; -import org.apache.cassandra.db.filter.NamesQueryFilter; -import org.apache.cassandra.db.filter.QueryFilter; -import org.apache.cassandra.db.marshal.CounterColumnType; -import org.apache.cassandra.io.sstable.format.SSTableReader; -import org.apache.cassandra.io.util.FileUtils; -import org.apache.cassandra.tracing.Tracing; -import org.apache.cassandra.utils.SearchIterator; -import org.apache.cassandra.utils.memory.HeapAllocator; - -public class CollationController -{ - private final ColumnFamilyStore cfs; - private final QueryFilter filter; - private final int gcBefore; - - private int sstablesIterated = 0; - - public CollationController(ColumnFamilyStore cfs, QueryFilter filter, int gcBefore) - { - this.cfs = cfs; - this.filter = filter; - this.gcBefore = gcBefore; - } - - public ColumnFamily getTopLevelColumns(boolean copyOnHeap) - { - return filter.filter instanceof NamesQueryFilter - && cfs.metadata.getDefaultValidator() != CounterColumnType.instance - ? collectTimeOrderedData(copyOnHeap) - : collectAllData(copyOnHeap); - } - - /** - * Collects data in order of recency, using the sstable maxtimestamp data. - * Once we have data for all requests columns that is newer than the newest remaining maxtimestamp, - * we stop. - */ - private ColumnFamily collectTimeOrderedData(boolean copyOnHeap) - { - final ColumnFamily container = ArrayBackedSortedColumns.factory.create(cfs.metadata, filter.filter.isReversed()); - List iterators = new ArrayList<>(); - boolean isEmpty = true; - Tracing.trace("Acquiring sstable references"); - ColumnFamilyStore.ViewFragment view = cfs.select(cfs.viewFilter(filter.key)); - DeletionInfo returnDeletionInfo = container.deletionInfo(); - - try - { - Tracing.trace("Merging memtable contents"); - for (Memtable memtable : view.memtables) - { - ColumnFamily cf = memtable.getColumnFamily(filter.key); - if (cf != null) - { - filter.delete(container.deletionInfo(), cf); - isEmpty = false; - Iterator iter = filter.getIterator(cf); - while (iter.hasNext()) - { - Cell cell = iter.next(); - if (copyOnHeap) - cell = cell.localCopy(cfs.metadata, HeapAllocator.instance); - container.addColumn(cell); - } - } - } - - // avoid changing the filter columns of the original filter - // (reduceNameFilter removes columns that are known to be irrelevant) - NamesQueryFilter namesFilter = (NamesQueryFilter) filter.filter; - TreeSet filterColumns = new TreeSet<>(namesFilter.columns); - QueryFilter reducedFilter = new QueryFilter(filter.key, filter.cfName, namesFilter.withUpdatedColumns(filterColumns), filter.timestamp); - - /* add the SSTables on disk */ - Collections.sort(view.sstables, SSTableReader.maxTimestampComparator); - - // read sorted sstables - for (SSTableReader sstable : view.sstables) - { - // if we've already seen a row tombstone with a timestamp greater - // than the most recent update to this sstable, we're done, since the rest of the sstables - // will also be older - if (sstable.getMaxTimestamp() < returnDeletionInfo.getTopLevelDeletion().markedForDeleteAt) - break; - - long currentMaxTs = sstable.getMaxTimestamp(); - reduceNameFilter(reducedFilter, container, currentMaxTs); - if (((NamesQueryFilter) reducedFilter.filter).columns.isEmpty()) - break; - - Tracing.trace("Merging data from sstable {}", sstable.descriptor.generation); - sstable.incrementReadCount(); - OnDiskAtomIterator iter = reducedFilter.getSSTableColumnIterator(sstable); - iterators.add(iter); - isEmpty = false; - if (iter.getColumnFamily() != null) - { - container.delete(iter.getColumnFamily()); - sstablesIterated++; - while (iter.hasNext()) - container.addAtom(iter.next()); - } - } - - // we need to distinguish between "there is no data at all for this row" (BF will let us rebuild that efficiently) - // and "there used to be data, but it's gone now" (we should cache the empty CF so we don't need to rebuild that slower) - if (isEmpty) - return null; - - // do a final collate. toCollate is boilerplate required to provide a CloseableIterator - ColumnFamily returnCF = container.cloneMeShallow(); - Tracing.trace("Collating all results"); - filter.collateOnDiskAtom(returnCF, container.iterator(), gcBefore); - - // "hoist up" the requested data into a more recent sstable - if (sstablesIterated > cfs.getMinimumCompactionThreshold() - && !cfs.isAutoCompactionDisabled() - && cfs.getCompactionStrategyManager().shouldDefragment()) - { - // !!WARNING!! if we stop copying our data to a heap-managed object, - // we will need to track the lifetime of this mutation as well - Tracing.trace("Defragmenting requested data"); - final Mutation mutation = new Mutation(cfs.keyspace.getName(), filter.key.getKey(), returnCF.cloneMe()); - StageManager.getStage(Stage.MUTATION).execute(new Runnable() - { - public void run() - { - // skipping commitlog and index updates is fine since we're just de-fragmenting existing data - Keyspace.open(mutation.getKeyspaceName()).apply(mutation, false, false); - } - }); - } - - // Caller is responsible for final removeDeletedCF. This is important for cacheRow to work correctly: - return returnCF; - } - finally - { - for (OnDiskAtomIterator iter : iterators) - FileUtils.closeQuietly(iter); - } - } - - /** - * remove columns from @param filter where we already have data in @param container newer than @param sstableTimestamp - */ - private void reduceNameFilter(QueryFilter filter, ColumnFamily container, long sstableTimestamp) - { - if (container == null) - return; - - SearchIterator searchIter = container.searchIterator(); - for (Iterator iterator = ((NamesQueryFilter) filter.filter).columns.iterator(); iterator.hasNext() && searchIter.hasNext(); ) - { - CellName filterColumn = iterator.next(); - Cell cell = searchIter.next(filterColumn); - if (cell != null && cell.timestamp() > sstableTimestamp) - iterator.remove(); - } - } - - /** - * Collects data the brute-force way: gets an iterator for the filter in question - * from every memtable and sstable, then merges them together. - */ - private ColumnFamily collectAllData(boolean copyOnHeap) - { - Tracing.trace("Acquiring sstable references"); - ColumnFamilyStore.ViewFragment view = cfs.select(cfs.viewFilter(filter.key)); - List> iterators = new ArrayList<>(Iterables.size(view.memtables) + view.sstables.size()); - ColumnFamily returnCF = ArrayBackedSortedColumns.factory.create(cfs.metadata, filter.filter.isReversed()); - DeletionInfo returnDeletionInfo = returnCF.deletionInfo(); - try - { - Tracing.trace("Merging memtable tombstones"); - for (Memtable memtable : view.memtables) - { - final ColumnFamily cf = memtable.getColumnFamily(filter.key); - if (cf != null) - { - filter.delete(returnDeletionInfo, cf); - Iterator iter = filter.getIterator(cf); - if (copyOnHeap) - { - iter = Iterators.transform(iter, new Function() - { - public Cell apply(Cell cell) - { - return cell.localCopy(cf.metadata, HeapAllocator.instance); - } - }); - } - iterators.add(iter); - } - } - - /* - * We can't eliminate full sstables based on the timestamp of what we've already read like - * in collectTimeOrderedData, but we still want to eliminate sstable whose maxTimestamp < mostRecentTombstone - * we've read. We still rely on the sstable ordering by maxTimestamp since if - * maxTimestamp_s1 > maxTimestamp_s0, - * we're guaranteed that s1 cannot have a row tombstone such that - * timestamp(tombstone) > maxTimestamp_s0 - * since we necessarily have - * timestamp(tombstone) <= maxTimestamp_s1 - * In other words, iterating in maxTimestamp order allow to do our mostRecentTombstone elimination - * in one pass, and minimize the number of sstables for which we read a rowTombstone. - */ - Collections.sort(view.sstables, SSTableReader.maxTimestampComparator); - List skippedSSTables = null; - long minTimestamp = Long.MAX_VALUE; - int nonIntersectingSSTables = 0; - - for (SSTableReader sstable : view.sstables) - { - minTimestamp = Math.min(minTimestamp, sstable.getMinTimestamp()); - // if we've already seen a row tombstone with a timestamp greater - // than the most recent update to this sstable, we can skip it - if (sstable.getMaxTimestamp() < returnDeletionInfo.getTopLevelDeletion().markedForDeleteAt) - break; - - if (!filter.shouldInclude(sstable)) - { - nonIntersectingSSTables++; - // sstable contains no tombstone if maxLocalDeletionTime == Integer.MAX_VALUE, so we can safely skip those entirely - if (sstable.getSSTableMetadata().maxLocalDeletionTime != Integer.MAX_VALUE) - { - if (skippedSSTables == null) - skippedSSTables = new ArrayList<>(); - skippedSSTables.add(sstable); - } - continue; - } - - sstable.incrementReadCount(); - OnDiskAtomIterator iter = filter.getSSTableColumnIterator(sstable); - iterators.add(iter); - if (iter.getColumnFamily() != null) - { - ColumnFamily cf = iter.getColumnFamily(); - returnCF.delete(cf); - sstablesIterated++; - } - } - - int includedDueToTombstones = 0; - // Check for row tombstone in the skipped sstables - if (skippedSSTables != null) - { - for (SSTableReader sstable : skippedSSTables) - { - if (sstable.getMaxTimestamp() <= minTimestamp) - continue; - - sstable.incrementReadCount(); - OnDiskAtomIterator iter = filter.getSSTableColumnIterator(sstable); - ColumnFamily cf = iter.getColumnFamily(); - // we are only interested in row-level tombstones here, and only if markedForDeleteAt is larger than minTimestamp - if (cf != null && cf.deletionInfo().getTopLevelDeletion().markedForDeleteAt > minTimestamp) - { - includedDueToTombstones++; - iterators.add(iter); - returnCF.delete(cf.deletionInfo().getTopLevelDeletion()); - sstablesIterated++; - } - else - { - FileUtils.closeQuietly(iter); - } - } - } - - if (Tracing.isTracing()) - Tracing.trace("Skipped {}/{} non-slice-intersecting sstables, included {} due to tombstones", - nonIntersectingSSTables, view.sstables.size(), includedDueToTombstones); - - // we need to distinguish between "there is no data at all for this row" (BF will let us rebuild that efficiently) - // and "there used to be data, but it's gone now" (we should cache the empty CF so we don't need to rebuild that slower) - if (iterators.isEmpty()) - return null; - - Tracing.trace("Merging data from memtables and {} sstables", sstablesIterated); - filter.collateOnDiskAtom(returnCF, iterators, gcBefore); - - // Caller is responsible for final removeDeletedCF. This is important for cacheRow to work correctly: - return returnCF; - } - finally - { - for (Object iter : iterators) - if (iter instanceof Closeable) - FileUtils.closeQuietly((Closeable) iter); - } - } - - public int getSstablesIterated() - { - return sstablesIterated; - } -} diff --git a/src/java/org/apache/cassandra/db/ColumnFamily.java b/src/java/org/apache/cassandra/db/ColumnFamily.java deleted file mode 100644 index a7243a2fba..0000000000 --- a/src/java/org/apache/cassandra/db/ColumnFamily.java +++ /dev/null @@ -1,565 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.apache.cassandra.db; - -import java.io.DataInputStream; -import java.io.IOException; -import java.nio.ByteBuffer; -import java.security.MessageDigest; -import java.util.Collection; -import java.util.Collections; -import java.util.Iterator; -import java.util.List; -import java.util.Map; -import java.util.UUID; - -import com.google.common.collect.ImmutableMap; -import org.apache.commons.lang3.builder.HashCodeBuilder; - -import org.apache.cassandra.cache.IRowCacheEntry; -import org.apache.cassandra.config.CFMetaData; -import org.apache.cassandra.config.Schema; -import org.apache.cassandra.db.composites.CellName; -import org.apache.cassandra.db.composites.CellNameType; -import org.apache.cassandra.db.composites.CellNames; -import org.apache.cassandra.db.filter.ColumnCounter; -import org.apache.cassandra.db.filter.ColumnSlice; -import org.apache.cassandra.io.sstable.ColumnNameHelper; -import org.apache.cassandra.io.sstable.ColumnStats; -import org.apache.cassandra.io.sstable.SSTable; -import org.apache.cassandra.io.util.DataOutputBuffer; -import org.apache.cassandra.net.MessagingService; -import org.apache.cassandra.utils.*; - -/** - * A sorted map of columns. - * This represents the backing map of a colum family. - * - * Whether the implementation is thread safe or not is left to the - * implementing classes. - */ -public abstract class ColumnFamily implements Iterable, IRowCacheEntry -{ - /* The column serializer for this Column Family. Create based on config. */ - public static final ColumnFamilySerializer serializer = new ColumnFamilySerializer(); - - protected final CFMetaData metadata; - - protected ColumnFamily(CFMetaData metadata) - { - assert metadata != null; - this.metadata = metadata; - } - - public T cloneMeShallow(ColumnFamily.Factory factory, boolean reversedInsertOrder) - { - T cf = factory.create(metadata, reversedInsertOrder); - cf.delete(this); - return cf; - } - - public ColumnFamily cloneMeShallow() - { - return cloneMeShallow(false); - } - - public ColumnFamily cloneMeShallow(boolean reversed) - { - return cloneMeShallow(getFactory(), reversed); - } - - public ColumnFamilyType getType() - { - return metadata.cfType; - } - - public int liveCQL3RowCount(long now) - { - ColumnCounter counter = getComparator().isDense() - ? new ColumnCounter(now) - : new ColumnCounter.GroupByPrefix(now, getComparator(), metadata.clusteringColumns().size()); - return counter.countAll(this).live(); - } - - /** - * Clones the column map. - */ - public abstract ColumnFamily cloneMe(); - - public UUID id() - { - return metadata.cfId; - } - - /** - * @return The CFMetaData for this row - */ - public CFMetaData metadata() - { - return metadata; - } - - public void addColumn(CellName name, ByteBuffer value, long timestamp) - { - addColumn(name, value, timestamp, 0); - } - - public void addColumn(CellName name, ByteBuffer value, long timestamp, int timeToLive) - { - assert !metadata().isCounter(); - Cell cell = AbstractCell.create(name, value, timestamp, timeToLive, metadata()); - addColumn(cell); - } - - public void addCounter(CellName name, long value) - { - addColumn(new BufferCounterUpdateCell(name, value, FBUtilities.timestampMicros())); - } - - public void addTombstone(CellName name, ByteBuffer localDeletionTime, long timestamp) - { - addColumn(new BufferDeletedCell(name, localDeletionTime, timestamp)); - } - - public void addTombstone(CellName name, int localDeletionTime, long timestamp) - { - addColumn(new BufferDeletedCell(name, localDeletionTime, timestamp)); - } - - public void addAtom(OnDiskAtom atom) - { - if (atom instanceof Cell) - { - addColumn((Cell)atom); - } - else - { - assert atom instanceof RangeTombstone; - delete((RangeTombstone)atom); - } - } - - /** - * Clear this column family, removing all columns and deletion info. - */ - public abstract void clear(); - - /** - * Returns a {@link DeletionInfo.InOrderTester} for the deletionInfo() of - * this column family. Please note that for ThreadSafe implementation of ColumnFamily, - * this tester will remain valid even if new tombstones are added to this ColumnFamily - * *as long as said addition is done in comparator order*. For AtomicSortedColumns, - * the tester will correspond to the state of when this method is called. - */ - public DeletionInfo.InOrderTester inOrderDeletionTester() - { - return deletionInfo().inOrderTester(); - } - - /** - * Returns the factory used for this ISortedColumns implementation. - */ - public abstract Factory getFactory(); - - public abstract DeletionInfo deletionInfo(); - public abstract void setDeletionInfo(DeletionInfo info); - - public abstract void delete(DeletionInfo info); - public abstract void delete(DeletionTime deletionTime); - protected abstract void delete(RangeTombstone tombstone); - - public abstract SearchIterator searchIterator(); - - /** - * Purges top-level and range tombstones whose localDeletionTime is older than gcBefore. - * @param gcBefore a timestamp (in seconds) before which tombstones should be purged - */ - public abstract void purgeTombstones(int gcBefore); - - /** - * Adds a cell to this cell map. - * If a cell with the same name is already present in the map, it will - * be replaced by the newly added cell. - */ - public abstract void addColumn(Cell cell); - - /** - * Adds a cell if it's non-gc-able and isn't shadowed by a partition/range tombstone with a higher timestamp. - * Requires that the cell to add is sorted strictly after the last cell in the container. - */ - public abstract void maybeAppendColumn(Cell cell, DeletionInfo.InOrderTester tester, int gcBefore); - - /** - * Appends a cell. Requires that the cell to add is sorted strictly after the last cell in the container. - */ - public abstract void appendColumn(Cell cell); - - /** - * Adds all the columns of a given column map to this column map. - * This is equivalent to: - * - * for (Cell c : cm) - * addColumn(c, ...); - * - * but is potentially faster. - */ - public abstract void addAll(ColumnFamily cm); - - /** - * Get a column given its name, returning null if the column is not - * present. - */ - public abstract Cell getColumn(CellName name); - - /** - * Returns an iterable with the names of columns in this column map in the same order - * as the underlying columns themselves. - */ - public abstract Iterable getColumnNames(); - - /** - * Returns the columns of this column map as a collection. - * The columns in the returned collection should be sorted as the columns - * in this map. - */ - public abstract Collection getSortedColumns(); - - /** - * Returns the columns of this column map as a collection. - * The columns in the returned collection should be sorted in reverse - * order of the columns in this map. - */ - public abstract Collection getReverseSortedColumns(); - - /** - * Returns the number of columns in this map. - */ - public abstract int getColumnCount(); - - /** - * Returns whether or not there are any columns present. - */ - public abstract boolean hasColumns(); - - /** - * Returns true if this contains no columns or deletion info - */ - public boolean isEmpty() - { - return deletionInfo().isLive() && !hasColumns(); - } - - /** - * Returns an iterator over the columns of this map that returns only the matching @param slices. - * The provided slices must be in order and must be non-overlapping. - */ - public abstract Iterator iterator(ColumnSlice[] slices); - - /** - * Returns a reversed iterator over the columns of this map that returns only the matching @param slices. - * The provided slices must be in reversed order and must be non-overlapping. - */ - public abstract Iterator reverseIterator(ColumnSlice[] slices); - - /** - * Returns if this map only support inserts in reverse order. - */ - public abstract boolean isInsertReversed(); - - /** - * If `columns` has any tombstones (top-level or range tombstones), they will be applied to this set of columns. - */ - public void delete(ColumnFamily columns) - { - delete(columns.deletionInfo()); - } - - /* - * This function will calculate the difference between 2 column families. - * The external input is assumed to be a superset of internal. - */ - public ColumnFamily diff(ColumnFamily cfComposite) - { - assert cfComposite.id().equals(id()); - ColumnFamily cfDiff = ArrayBackedSortedColumns.factory.create(metadata); - cfDiff.delete(cfComposite.deletionInfo()); - - // (don't need to worry about cfNew containing Columns that are shadowed by - // the delete tombstone, since cfNew was generated by CF.resolve, which - // takes care of those for us.) - for (Cell cellExternal : cfComposite) - { - CellName cName = cellExternal.name(); - Cell cellInternal = getColumn(cName); - if (cellInternal == null) - { - cfDiff.addColumn(cellExternal); - } - else - { - Cell cellDiff = cellInternal.diff(cellExternal); - if (cellDiff != null) - { - cfDiff.addColumn(cellDiff); - } - } - } - - cfDiff.setDeletionInfo(deletionInfo().diff(cfComposite.deletionInfo())); - - if (!cfDiff.isEmpty()) - return cfDiff; - - return null; - } - - public long dataSize() - { - long size = 0; - for (Cell cell : this) - size += cell.cellDataSize(); - return size; - } - - public long maxTimestamp() - { - long maxTimestamp = deletionInfo().maxTimestamp(); - for (Cell cell : this) - maxTimestamp = Math.max(maxTimestamp, cell.timestamp()); - return maxTimestamp; - } - - @Override - public int hashCode() - { - HashCodeBuilder builder = new HashCodeBuilder(373, 75437) - .append(metadata) - .append(deletionInfo()); - for (Cell cell : this) - builder.append(cell); - return builder.toHashCode(); - } - - @Override - public boolean equals(Object o) - { - if (this == o) - return true; - if (o == null || !(o instanceof ColumnFamily)) - return false; - - ColumnFamily comparison = (ColumnFamily) o; - - return metadata.equals(comparison.metadata) - && deletionInfo().equals(comparison.deletionInfo()) - && ByteBufferUtil.compareUnsigned(digest(this), digest(comparison)) == 0; - } - - @Override - public String toString() - { - StringBuilder sb = new StringBuilder("ColumnFamily("); - sb.append(metadata.cfName); - - if (isMarkedForDelete()) - sb.append(" -").append(deletionInfo()).append("-"); - - sb.append(" [").append(CellNames.getColumnsString(getComparator(), this)).append("])"); - return sb.toString(); - } - - public static ByteBuffer digest(ColumnFamily cf) - { - MessageDigest digest = FBUtilities.threadLocalMD5Digest(); - if (cf != null) - cf.updateDigest(digest); - return ByteBuffer.wrap(digest.digest()); - } - - public void updateDigest(MessageDigest digest) - { - for (Cell cell : this) - cell.updateDigest(digest); - - deletionInfo().updateDigest(digest); - } - - public static ColumnFamily diff(ColumnFamily cf1, ColumnFamily cf2) - { - if (cf1 == null) - return cf2; - return cf1.diff(cf2); - } - - public ColumnStats getColumnStats() - { - // note that we default to MIN_VALUE/MAX_VALUE here to be able to override them later in this method - // we are checking row/range tombstones and actual cells - there should always be data that overrides - // these with actual values - ColumnStats.MinLongTracker minTimestampTracker = new ColumnStats.MinLongTracker(Long.MIN_VALUE); - ColumnStats.MaxLongTracker maxTimestampTracker = new ColumnStats.MaxLongTracker(Long.MAX_VALUE); - StreamingHistogram tombstones = new StreamingHistogram(SSTable.TOMBSTONE_HISTOGRAM_BIN_SIZE); - ColumnStats.MaxIntTracker maxDeletionTimeTracker = new ColumnStats.MaxIntTracker(Integer.MAX_VALUE); - List minColumnNamesSeen = Collections.emptyList(); - List maxColumnNamesSeen = Collections.emptyList(); - boolean hasLegacyCounterShards = false; - - if (deletionInfo().getTopLevelDeletion().localDeletionTime < Integer.MAX_VALUE) - { - tombstones.update(deletionInfo().getTopLevelDeletion().localDeletionTime); - maxDeletionTimeTracker.update(deletionInfo().getTopLevelDeletion().localDeletionTime); - minTimestampTracker.update(deletionInfo().getTopLevelDeletion().markedForDeleteAt); - maxTimestampTracker.update(deletionInfo().getTopLevelDeletion().markedForDeleteAt); - } - Iterator it = deletionInfo().rangeIterator(); - while (it.hasNext()) - { - RangeTombstone rangeTombstone = it.next(); - tombstones.update(rangeTombstone.getLocalDeletionTime()); - minTimestampTracker.update(rangeTombstone.timestamp()); - maxTimestampTracker.update(rangeTombstone.timestamp()); - maxDeletionTimeTracker.update(rangeTombstone.getLocalDeletionTime()); - minColumnNamesSeen = ColumnNameHelper.minComponents(minColumnNamesSeen, rangeTombstone.min, metadata.comparator); - maxColumnNamesSeen = ColumnNameHelper.maxComponents(maxColumnNamesSeen, rangeTombstone.max, metadata.comparator); - } - - for (Cell cell : this) - { - minTimestampTracker.update(cell.timestamp()); - maxTimestampTracker.update(cell.timestamp()); - maxDeletionTimeTracker.update(cell.getLocalDeletionTime()); - - int deletionTime = cell.getLocalDeletionTime(); - if (deletionTime < Integer.MAX_VALUE) - tombstones.update(deletionTime); - minColumnNamesSeen = ColumnNameHelper.minComponents(minColumnNamesSeen, cell.name(), metadata.comparator); - maxColumnNamesSeen = ColumnNameHelper.maxComponents(maxColumnNamesSeen, cell.name(), metadata.comparator); - if (cell instanceof CounterCell) - hasLegacyCounterShards = hasLegacyCounterShards || ((CounterCell) cell).hasLegacyShards(); - } - return new ColumnStats(getColumnCount(), - minTimestampTracker.get(), - maxTimestampTracker.get(), - maxDeletionTimeTracker.get(), - tombstones, - minColumnNamesSeen, - maxColumnNamesSeen, - hasLegacyCounterShards); - } - - public boolean isMarkedForDelete() - { - return !deletionInfo().isLive(); - } - - /** - * @return the comparator whose sorting order the contained columns conform to - */ - public CellNameType getComparator() - { - return metadata.comparator; - } - - public boolean hasOnlyTombstones(long now) - { - for (Cell cell : this) - if (cell.isLive(now)) - return false; - return true; - } - - public Iterator iterator() - { - return getSortedColumns().iterator(); - } - - public Iterator reverseIterator() - { - return getReverseSortedColumns().iterator(); - } - - public Map asMap() - { - ImmutableMap.Builder builder = ImmutableMap.builder(); - for (Cell cell : this) - builder.put(cell.name(), cell.value()); - return builder.build(); - } - - public static ColumnFamily fromBytes(ByteBuffer bytes) - { - if (bytes == null) - return null; - - try - { - return serializer.deserialize(new DataInputStream(ByteBufferUtil.inputStream(bytes)), - ArrayBackedSortedColumns.factory, - ColumnSerializer.Flag.LOCAL, - MessagingService.current_version); - } - catch (IOException e) - { - throw new RuntimeException(e); - } - } - - public ByteBuffer toBytes() - { - try (DataOutputBuffer out = new DataOutputBuffer()) - { - serializer.serialize(this, out, MessagingService.current_version); - return ByteBuffer.wrap(out.getData(), 0, out.getLength()); - } - } - - - /** - * @return an iterator where the removes are carried out once everything has been iterated - */ - public abstract BatchRemoveIterator batchRemoveIterator(); - - public abstract static class Factory - { - /** - * Returns a (initially empty) column map whose columns are sorted - * according to the provided comparator. - * The {@code insertReversed} flag is an hint on how we expect insertion to be perfomed, - * either in sorted or reverse sorted order. This is used by ArrayBackedSortedColumns to - * allow optimizing for both forward and reversed slices. This does not matter for ThreadSafeSortedColumns. - * Note that this is only an hint on how we expect to do insertion, this does not change the map sorting. - */ - public abstract T create(CFMetaData metadata, boolean insertReversed, int initialCapacity); - - public T create(CFMetaData metadata, boolean insertReversed) - { - return create(metadata, insertReversed, 0); - } - - public T create(CFMetaData metadata) - { - return create(metadata, false); - } - - public T create(String keyspace, String cfName) - { - return create(Schema.instance.getCFMetaData(keyspace, cfName)); - } - } - -} diff --git a/src/java/org/apache/cassandra/db/ColumnFamilySerializer.java b/src/java/org/apache/cassandra/db/ColumnFamilySerializer.java deleted file mode 100644 index 928c21f134..0000000000 --- a/src/java/org/apache/cassandra/db/ColumnFamilySerializer.java +++ /dev/null @@ -1,172 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.apache.cassandra.db; - -import java.io.DataInput; -import java.io.IOException; -import java.util.UUID; - -import org.apache.cassandra.config.Schema; -import org.apache.cassandra.io.ISSTableSerializer; -import org.apache.cassandra.io.IVersionedSerializer; -import org.apache.cassandra.io.sstable.format.Version; -import org.apache.cassandra.io.util.DataOutputPlus; -import org.apache.cassandra.net.MessagingService; -import org.apache.cassandra.utils.UUIDSerializer; - -public class ColumnFamilySerializer implements IVersionedSerializer, ISSTableSerializer -{ - /* - * Serialized ColumnFamily format: - * - * [serialized for intra-node writes only, e.g. returning a query result] - * - * - * - * [in sstable only] - * - * - * - * [always present] - * - * - * - * - */ - public void serialize(ColumnFamily cf, DataOutputPlus out, int version) - { - try - { - if (cf == null) - { - out.writeBoolean(false); - return; - } - - out.writeBoolean(true); - serializeCfId(cf.id(), out, version); - cf.getComparator().deletionInfoSerializer().serialize(cf.deletionInfo(), out, version); - ColumnSerializer columnSerializer = cf.getComparator().columnSerializer(); - int count = cf.getColumnCount(); - out.writeInt(count); - int written = 0; - for (Cell cell : cf) - { - columnSerializer.serialize(cell, out); - written++; - } - assert count == written: "Table had " + count + " columns, but " + written + " written"; - } - catch (IOException e) - { - throw new RuntimeException(e); - } - } - - public ColumnFamily deserialize(DataInput in, int version) throws IOException - { - return deserialize(in, ColumnSerializer.Flag.LOCAL, version); - } - - public ColumnFamily deserialize(DataInput in, ColumnSerializer.Flag flag, int version) throws IOException - { - return deserialize(in, ArrayBackedSortedColumns.factory, flag, version); - } - - public ColumnFamily deserialize(DataInput in, ColumnFamily.Factory factory, ColumnSerializer.Flag flag, int version) throws IOException - { - if (!in.readBoolean()) - return null; - - ColumnFamily cf = factory.create(Schema.instance.getCFMetaData(deserializeCfId(in, version))); - - if (cf.metadata().isSuper() && version < MessagingService.VERSION_20) - { - SuperColumns.deserializerSuperColumnFamily(in, cf, flag, version); - } - else - { - cf.delete(cf.getComparator().deletionInfoSerializer().deserialize(in, version)); - - ColumnSerializer columnSerializer = cf.getComparator().columnSerializer(); - int size = in.readInt(); - for (int i = 0; i < size; ++i) - cf.addColumn(columnSerializer.deserialize(in, flag)); - } - return cf; - } - - public long contentSerializedSize(ColumnFamily cf, TypeSizes typeSizes, int version) - { - long size = cf.getComparator().deletionInfoSerializer().serializedSize(cf.deletionInfo(), typeSizes, version); - size += typeSizes.sizeof(cf.getColumnCount()); - ColumnSerializer columnSerializer = cf.getComparator().columnSerializer(); - for (Cell cell : cf) - size += columnSerializer.serializedSize(cell, typeSizes); - return size; - } - - public long serializedSize(ColumnFamily cf, TypeSizes typeSizes, int version) - { - if (cf == null) - { - return typeSizes.sizeof(false); - } - else - { - return typeSizes.sizeof(true) /* nullness bool */ - + cfIdSerializedSize(cf.id(), typeSizes, version) /* id */ - + contentSerializedSize(cf, typeSizes, version); - } - } - - public long serializedSize(ColumnFamily cf, int version) - { - return serializedSize(cf, TypeSizes.NATIVE, version); - } - - public void serializeForSSTable(ColumnFamily cf, DataOutputPlus out) - { - // Column families shouldn't be written directly to disk, use ColumnIndex.Builder instead - throw new UnsupportedOperationException(); - } - - public ColumnFamily deserializeFromSSTable(DataInput in, Version version) - { - throw new UnsupportedOperationException(); - } - - public void serializeCfId(UUID cfId, DataOutputPlus out, int version) throws IOException - { - UUIDSerializer.serializer.serialize(cfId, out, version); - } - - public UUID deserializeCfId(DataInput in, int version) throws IOException - { - UUID cfId = UUIDSerializer.serializer.deserialize(in, version); - if (Schema.instance.getCF(cfId) == null) - throw new UnknownColumnFamilyException("Couldn't find cfId=" + cfId, cfId); - - return cfId; - } - - public int cfIdSerializedSize(UUID cfId, TypeSizes typeSizes, int version) - { - return typeSizes.sizeof(cfId); - } -} diff --git a/src/java/org/apache/cassandra/db/ColumnFamilyStore.java b/src/java/org/apache/cassandra/db/ColumnFamilyStore.java index bed5d39bf9..8fb83afd62 100644 --- a/src/java/org/apache/cassandra/db/ColumnFamilyStore.java +++ b/src/java/org/apache/cassandra/db/ColumnFamilyStore.java @@ -47,19 +47,14 @@ import org.apache.cassandra.cache.*; import org.apache.cassandra.concurrent.*; import org.apache.cassandra.config.*; import org.apache.cassandra.config.CFMetaData.SpeculativeRetry; +import org.apache.cassandra.db.rows.*; import org.apache.cassandra.db.commitlog.CommitLog; import org.apache.cassandra.db.commitlog.ReplayPosition; import org.apache.cassandra.db.compaction.*; -import org.apache.cassandra.db.composites.CellName; -import org.apache.cassandra.db.composites.CellNameType; -import org.apache.cassandra.db.composites.Composite; -import org.apache.cassandra.db.filter.ColumnSlice; -import org.apache.cassandra.db.filter.ExtendedFilter; -import org.apache.cassandra.db.filter.IDiskAtomFilter; -import org.apache.cassandra.db.filter.QueryFilter; -import org.apache.cassandra.db.filter.SliceQueryFilter; +import org.apache.cassandra.db.filter.*; import org.apache.cassandra.db.index.SecondaryIndex; import org.apache.cassandra.db.index.SecondaryIndexManager; +import org.apache.cassandra.db.partitions.*; import org.apache.cassandra.dht.*; import org.apache.cassandra.dht.Range; import org.apache.cassandra.exceptions.ConfigurationException; @@ -76,7 +71,6 @@ import org.apache.cassandra.metrics.ColumnFamilyMetrics.Sampler; import org.apache.cassandra.service.CacheService; import org.apache.cassandra.service.StorageService; import org.apache.cassandra.streaming.StreamLockfile; -import org.apache.cassandra.tracing.Tracing; import org.apache.cassandra.utils.*; import org.apache.cassandra.utils.concurrent.*; import org.apache.cassandra.utils.TopKSampler.SamplerResult; @@ -592,12 +586,9 @@ public class ColumnFamilyStore implements ColumnFamilyStoreMBean { if (def.isIndexed()) { - CellNameType indexComparator = SecondaryIndex.getIndexComparator(metadata, def); - if (indexComparator != null) - { - CFMetaData indexMetadata = CFMetaData.newIndexMetadata(metadata, def, indexComparator); + CFMetaData indexMetadata = SecondaryIndex.newIndexMetadata(metadata, def); + if (indexMetadata != null) scrubDataDirectories(indexMetadata); - } } } } @@ -1220,7 +1211,7 @@ public class ColumnFamilyStore implements ColumnFamilyStoreMBean return; RowCacheKey cacheKey = new RowCacheKey(metadata.cfId, key); - invalidateCachedRow(cacheKey); + invalidateCachedPartition(cacheKey); } /** @@ -1230,11 +1221,12 @@ public class ColumnFamilyStore implements ColumnFamilyStoreMBean * param @ key - key for update/insert * param @ columnFamily - columnFamily changes */ - public void apply(DecoratedKey key, ColumnFamily columnFamily, SecondaryIndexManager.Updater indexer, OpOrder.Group opGroup, ReplayPosition replayPosition) + public void apply(PartitionUpdate update, SecondaryIndexManager.Updater indexer, OpOrder.Group opGroup, ReplayPosition replayPosition) { long start = System.nanoTime(); Memtable mt = data.getMemtableFor(opGroup, replayPosition); - final long timeDelta = mt.put(key, columnFamily, indexer, opGroup); + long timeDelta = mt.put(update, indexer, opGroup); + DecoratedKey key = update.partitionKey(); maybeUpdateRowCache(key); metric.samplers.get(Sampler.WRITES).addSample(key.getKey(), key.hashCode(), 1); metric.writeLatency.addNano(System.nanoTime() - start); @@ -1242,93 +1234,6 @@ public class ColumnFamilyStore implements ColumnFamilyStoreMBean metric.colUpdateTimeDeltaHistogram.update(timeDelta); } - /** - * Purges gc-able top-level and range tombstones, returning `cf` if there are any columns or tombstones left, - * null otherwise. - * @param gcBefore a timestamp (in seconds); tombstones with a localDeletionTime before this will be purged - */ - public static ColumnFamily removeDeletedCF(ColumnFamily cf, int gcBefore) - { - // purge old top-level and range tombstones - cf.purgeTombstones(gcBefore); - - // if there are no columns or tombstones left, return null - return !cf.hasColumns() && !cf.isMarkedForDelete() ? null : cf; - } - - /** - * Removes deleted columns and purges gc-able tombstones. - * @return an updated `cf` if any columns or tombstones remain, null otherwise - */ - public static ColumnFamily removeDeleted(ColumnFamily cf, int gcBefore) - { - return removeDeleted(cf, gcBefore, SecondaryIndexManager.nullUpdater); - } - - /* - This is complicated because we need to preserve deleted columns and columnfamilies - until they have been deleted for at least GC_GRACE_IN_SECONDS. But, we do not need to preserve - their contents; just the object itself as a "tombstone" that can be used to repair other - replicas that do not know about the deletion. - */ - public static ColumnFamily removeDeleted(ColumnFamily cf, int gcBefore, SecondaryIndexManager.Updater indexer) - { - if (cf == null) - { - return null; - } - - return removeDeletedCF(removeDeletedColumnsOnly(cf, gcBefore, indexer), gcBefore); - } - - /** - * Removes only per-cell tombstones, cells that are shadowed by a row-level or range tombstone, or - * columns that have been dropped from the schema (for CQL3 tables only). - * @return the updated ColumnFamily - */ - public static ColumnFamily removeDeletedColumnsOnly(ColumnFamily cf, int gcBefore, SecondaryIndexManager.Updater indexer) - { - BatchRemoveIterator iter = cf.batchRemoveIterator(); - DeletionInfo.InOrderTester tester = cf.inOrderDeletionTester(); - boolean hasDroppedColumns = !cf.metadata.getDroppedColumns().isEmpty(); - while (iter.hasNext()) - { - Cell c = iter.next(); - // remove columns if - // (a) the column itself is gcable or - // (b) the column is shadowed by a CF tombstone - // (c) the column has been dropped from the CF schema (CQL3 tables only) - if (c.getLocalDeletionTime() < gcBefore || tester.isDeleted(c) || (hasDroppedColumns && isDroppedColumn(c, cf.metadata()))) - { - iter.remove(); - indexer.remove(c); - } - } - iter.commit(); - return cf; - } - - // returns true if - // 1. this column has been dropped from schema and - // 2. if it has been re-added since then, this particular column was inserted before the last drop - private static boolean isDroppedColumn(Cell c, CFMetaData meta) - { - Long droppedAt = meta.getDroppedColumns().get(c.name().cql3ColumnName(meta)); - return droppedAt != null && c.timestamp() <= droppedAt; - } - - private void removeDroppedColumns(ColumnFamily cf) - { - if (cf == null || cf.metadata.getDroppedColumns().isEmpty()) - return; - - BatchRemoveIterator iter = cf.batchRemoveIterator(); - while (iter.hasNext()) - if (isDroppedColumn(iter.next(), metadata)) - iter.remove(); - iter.commit(); - } - /** * @param sstables * @return sstables whose key range overlaps with that of the given sstables, not including itself. @@ -1348,7 +1253,7 @@ public class ColumnFamilyStore implements ColumnFamilyStoreMBean Set results = null; for (SSTableReader sstable : sstables) { - Set overlaps = ImmutableSet.copyOf(tree.search(Interval.create(sstable.first, sstable.last))); + Set overlaps = ImmutableSet.copyOf(tree.search(Interval.create(sstable.first, sstable.last))); results = results == null ? overlaps : Sets.union(results, overlaps).immutableCopy(); } results = Sets.difference(results, ImmutableSet.copyOf(sstables)); @@ -1532,9 +1437,6 @@ public class ColumnFamilyStore implements ColumnFamilyStoreMBean return valid; } - - - /** * Package protected for access from the CompactionManager. */ @@ -1553,249 +1455,30 @@ public class ColumnFamilyStore implements ColumnFamilyStoreMBean return data.getUncompacting(); } - public ColumnFamily getColumnFamily(DecoratedKey key, - Composite start, - Composite finish, - boolean reversed, - int limit, - long timestamp) - { - return getColumnFamily(QueryFilter.getSliceFilter(key, name, start, finish, reversed, limit, timestamp)); - } - - /** - * Fetch the row and columns given by filter.key if it is in the cache; if not, read it from disk and cache it - * - * If row is cached, and the filter given is within its bounds, we return from cache, otherwise from disk - * - * If row is not cached, we figure out what filter is "biggest", read that from disk, then - * filter the result and either cache that or return it. - * - * @param cfId the column family to read the row from - * @param filter the columns being queried. - * @return the requested data for the filter provided - */ - private ColumnFamily getThroughCache(UUID cfId, QueryFilter filter) - { - assert isRowCacheEnabled() - : String.format("Row cache is not enabled on table [" + name + "]"); - - RowCacheKey key = new RowCacheKey(cfId, filter.key); - - // attempt a sentinel-read-cache sequence. if a write invalidates our sentinel, we'll return our - // (now potentially obsolete) data, but won't cache it. see CASSANDRA-3862 - // TODO: don't evict entire rows on writes (#2864) - IRowCacheEntry cached = CacheService.instance.rowCache.get(key); - if (cached != null) - { - if (cached instanceof RowCacheSentinel) - { - // Some other read is trying to cache the value, just do a normal non-caching read - Tracing.trace("Row cache miss (race)"); - metric.rowCacheMiss.inc(); - return getTopLevelColumns(filter, Integer.MIN_VALUE); - } - - ColumnFamily cachedCf = (ColumnFamily)cached; - if (isFilterFullyCoveredBy(filter.filter, cachedCf, filter.timestamp)) - { - metric.rowCacheHit.inc(); - Tracing.trace("Row cache hit"); - return filterColumnFamily(cachedCf, filter); - } - - metric.rowCacheHitOutOfRange.inc(); - Tracing.trace("Ignoring row cache as cached value could not satisfy query"); - return getTopLevelColumns(filter, Integer.MIN_VALUE); - } - - metric.rowCacheMiss.inc(); - Tracing.trace("Row cache miss"); - RowCacheSentinel sentinel = new RowCacheSentinel(); - boolean sentinelSuccess = CacheService.instance.rowCache.putIfAbsent(key, sentinel); - ColumnFamily data = null; - ColumnFamily toCache = null; - try - { - // If we are explicitely asked to fill the cache with full partitions, we go ahead and query the whole thing - if (metadata.getCaching().rowCache.cacheFullPartitions()) - { - data = getTopLevelColumns(QueryFilter.getIdentityFilter(filter.key, name, filter.timestamp), Integer.MIN_VALUE); - toCache = data; - Tracing.trace("Populating row cache with the whole partition"); - if (sentinelSuccess && toCache != null) - CacheService.instance.rowCache.replace(key, sentinel, toCache); - return filterColumnFamily(data, filter); - } - - // Otherwise, if we want to cache the result of the query we're about to do, we must make sure this query - // covers what needs to be cached. And if the user filter does not satisfy that, we sometimes extend said - // filter so we can populate the cache but only if: - // 1) we can guarantee it is a strict extension, i.e. that we will still fetch the data asked by the user. - // 2) the extension does not make us query more than getRowsPerPartitionToCache() (as a mean to limit the - // amount of extra work we'll do on a user query for the purpose of populating the cache). - // - // In practice, we can only guarantee those 2 points if the filter is one that queries the head of the - // partition (and if that filter actually counts CQL3 rows since that's what we cache and it would be - // bogus to compare the filter count to the 'rows to cache' otherwise). - if (filter.filter.isHeadFilter() && filter.filter.countCQL3Rows(metadata.comparator)) - { - SliceQueryFilter sliceFilter = (SliceQueryFilter)filter.filter; - int rowsToCache = metadata.getCaching().rowCache.rowsToCache; - - SliceQueryFilter cacheSlice = readFilterForCache(); - QueryFilter cacheFilter = new QueryFilter(filter.key, name, cacheSlice, filter.timestamp); - - // If the filter count is less than the number of rows cached, we simply extend it to make sure we do cover the - // number of rows to cache, and if that count is greater than the number of rows to cache, we simply filter what - // needs to be cached afterwards. - if (sliceFilter.count < rowsToCache) - { - toCache = getTopLevelColumns(cacheFilter, Integer.MIN_VALUE); - if (toCache != null) - { - Tracing.trace("Populating row cache ({} rows cached)", cacheSlice.lastCounted()); - data = filterColumnFamily(toCache, filter); - } - } - else - { - data = getTopLevelColumns(filter, Integer.MIN_VALUE); - if (data != null) - { - // The filter limit was greater than the number of rows to cache. But, if the filter had a non-empty - // finish bound, we may have gotten less than what needs to be cached, in which case we shouldn't cache it - // (otherwise a cache hit would assume the whole partition is cached which is not the case). - if (sliceFilter.finish().isEmpty() || sliceFilter.lastCounted() >= rowsToCache) - { - toCache = filterColumnFamily(data, cacheFilter); - Tracing.trace("Caching {} rows (out of {} requested)", cacheSlice.lastCounted(), sliceFilter.count); - } - else - { - Tracing.trace("Not populating row cache, not enough rows fetched ({} fetched but {} required for the cache)", sliceFilter.lastCounted(), rowsToCache); - } - } - } - - if (sentinelSuccess && toCache != null) - CacheService.instance.rowCache.replace(key, sentinel, toCache); - return data; - } - else - { - Tracing.trace("Fetching data but not populating cache as query does not query from the start of the partition"); - return getTopLevelColumns(filter, Integer.MIN_VALUE); - } - } - finally - { - if (sentinelSuccess && toCache == null) - invalidateCachedRow(key); - } - } - - public SliceQueryFilter readFilterForCache() - { - // We create a new filter everytime before for now SliceQueryFilter is unfortunatly mutable. - return new SliceQueryFilter(ColumnSlice.ALL_COLUMNS_ARRAY, false, metadata.getCaching().rowCache.rowsToCache, metadata.clusteringColumns().size()); - } - - public boolean isFilterFullyCoveredBy(IDiskAtomFilter filter, ColumnFamily cachedCf, long now) + public boolean isFilterFullyCoveredBy(ClusteringIndexFilter filter, DataLimits limits, CachedPartition cached, int nowInSec) { // We can use the cached value only if we know that no data it doesn't contain could be covered // by the query filter, that is if: // 1) either the whole partition is cached - // 2) or we can ensure than any data the filter selects are in the cached partition + // 2) or we can ensure than any data the filter selects is in the cached partition - // When counting rows to decide if the whole row is cached, we should be careful with expiring - // columns: if we use a timestamp newer than the one that was used when populating the cache, we might - // end up deciding the whole partition is cached when it's really not (just some rows expired since the - // cf was cached). This is the reason for Integer.MIN_VALUE below. - boolean wholePartitionCached = cachedCf.liveCQL3RowCount(Integer.MIN_VALUE) < metadata.getCaching().rowCache.rowsToCache; + // We can guarantee that a partition is fully cached if the number of rows it contains is less than + // what we're caching. Wen doing that, we should be careful about expiring cells: we should count + // something expired that wasn't when the partition was cached, or we could decide that the whole + // partition is cached when it's not. This is why we use CachedPartition#cachedLiveRows. + if (cached.cachedLiveRows() < metadata.getCaching().rowCache.rowsToCache) + return true; - // Contrarily to the "wholePartitionCached" check above, we do want isFullyCoveredBy to take the - // timestamp of the query into account when dealing with expired columns. Otherwise, we could think - // the cached partition has enough live rows to satisfy the filter when it doesn't because some - // are now expired. - return wholePartitionCached || filter.isFullyCoveredBy(cachedCf, now); + // If the whole partition isn't cached, then we must guarantee that the filter cannot select data that + // is not in the cache. We can guarantee that if either the filter is a "head filter" and the cached + // partition has more live rows that queried (where live rows refers to the rows that are live now), + // or if we can prove that everything the filter selects is in the cached partition based on its content. + return (filter.isHeadFilter() && limits.hasEnoughLiveData(cached, nowInSec)) || filter.isFullyCoveredBy(cached); } - public int gcBefore(long now) + public int gcBefore(int nowInSec) { - return (int) (now / 1000) - metadata.getGcGraceSeconds(); - } - - /** - * get a list of columns starting from a given column, in a specified order. - * only the latest version of a column is returned. - * @return null if there is no data and no tombstones; otherwise a ColumnFamily - */ - public ColumnFamily getColumnFamily(QueryFilter filter) - { - assert name.equals(filter.getColumnFamilyName()) : filter.getColumnFamilyName(); - - ColumnFamily result = null; - - long start = System.nanoTime(); - try - { - int gcBefore = gcBefore(filter.timestamp); - if (isRowCacheEnabled()) - { - assert !isIndex(); // CASSANDRA-5732 - UUID cfId = metadata.cfId; - - ColumnFamily cached = getThroughCache(cfId, filter); - if (cached == null) - { - logger.trace("cached row is empty"); - return null; - } - - result = cached; - } - else - { - ColumnFamily cf = getTopLevelColumns(filter, gcBefore); - - if (cf == null) - return null; - - result = removeDeletedCF(cf, gcBefore); - } - - removeDroppedColumns(result); - - if (filter.filter instanceof SliceQueryFilter) - { - // Log the number of tombstones scanned on single key queries - metric.tombstoneScannedHistogram.update(((SliceQueryFilter) filter.filter).lastTombstones()); - metric.liveScannedHistogram.update(((SliceQueryFilter) filter.filter).lastLive()); - } - } - finally - { - metric.readLatency.addNano(System.nanoTime() - start); - } - - return result; - } - - /** - * Filter a cached row, which will not be modified by the filter, but may be modified by throwing out - * tombstones that are no longer relevant. - * The returned column family won't be thread safe. - */ - ColumnFamily filterColumnFamily(ColumnFamily cached, QueryFilter filter) - { - if (cached == null) - return null; - - ColumnFamily cf = cached.cloneMeShallow(ArrayBackedSortedColumns.factory, filter.filter.isReversed()); - int gcBefore = gcBefore(filter.timestamp); - filter.collateOnDiskAtom(cf, filter.getIterator(cached), gcBefore); - return removeDeletedCF(cf, gcBefore); + return nowInSec - metadata.getGcGraceSeconds(); } public Set getUnrepairedSSTables() @@ -1881,7 +1564,7 @@ public class ColumnFamilyStore implements ColumnFamilyStoreMBean * @return a ViewFragment containing the sstables and memtables that may need to be merged * for rows within @param rowBounds, inclusive, according to the interval tree. */ - public Function> viewFilter(final AbstractBounds rowBounds) + public Function> viewFilter(final AbstractBounds rowBounds) { return new Function>() { @@ -1896,14 +1579,14 @@ public class ColumnFamilyStore implements ColumnFamilyStoreMBean * @return a ViewFragment containing the sstables and memtables that may need to be merged * for rows for all of @param rowBoundsCollection, inclusive, according to the interval tree. */ - public Function> viewFilter(final Collection> rowBoundsCollection, final boolean includeRepaired) + public Function> viewFilter(final Collection> rowBoundsCollection, final boolean includeRepaired) { return new Function>() { public List apply(View view) { Set sstables = Sets.newHashSet(); - for (AbstractBounds rowBounds : rowBoundsCollection) + for (AbstractBounds rowBounds : rowBoundsCollection) { for (SSTableReader sstable : view.sstablesInBounds(rowBounds)) { @@ -1934,20 +1617,6 @@ public class ColumnFamilyStore implements ColumnFamilyStoreMBean } } - public ColumnFamily getTopLevelColumns(QueryFilter filter, int gcBefore) - { - Tracing.trace("Executing single-partition query on {}", name); - CollationController controller = new CollationController(this, filter, gcBefore); - ColumnFamily columns; - try (OpOrder.Group op = readOrdering.start()) - { - columns = controller.getTopLevelColumns(Memtable.MEMORY_POOL.needToCopyOnHeap()); - } - if (columns != null) - metric.samplers.get(Sampler.READS).addSample(filter.key.getKey(), filter.key.hashCode(), 1); - metric.updateSSTableIterated(controller.getSstablesIterated()); - return columns; - } public void beginLocalSampling(String sampler, int capacity) { @@ -1982,7 +1651,7 @@ public class ColumnFamilyStore implements ColumnFamilyStoreMBean RowCacheKey key = keyIter.next(); DecoratedKey dk = partitioner.decorateKey(ByteBuffer.wrap(key.key)); if (key.cfId.equals(metadata.cfId) && !Range.isInRanges(dk.getToken(), ranges)) - invalidateCachedRow(dk); + invalidateCachedPartition(dk); } if (metadata.isCounter()) @@ -1998,247 +1667,7 @@ public class ColumnFamilyStore implements ColumnFamilyStoreMBean } } - public static abstract class AbstractScanIterator extends AbstractIterator implements CloseableIterator - { - public boolean needsFiltering() - { - return true; - } - } - - /** - * Iterate over a range of rows and columns from memtables/sstables. - * - * @param range The range of keys and columns within those keys to fetch - */ - @SuppressWarnings("resource") - private AbstractScanIterator getSequentialIterator(final DataRange range, long now) - { - assert !(range.keyRange() instanceof Range) || !((Range)range.keyRange()).isWrapAround() || range.keyRange().right.isMinimum() : range.keyRange(); - - final ViewFragment view = select(viewFilter(range.keyRange())); - Tracing.trace("Executing seq scan across {} sstables for {}", view.sstables.size(), range.keyRange().getString(metadata.getKeyValidator())); - - final CloseableIterator iterator = RowIteratorFactory.getIterator(view.memtables, view.sstables, range, this, now); - - // todo this could be pushed into SSTableScanner - return new AbstractScanIterator() - { - protected Row computeNext() - { - while (true) - { - // pull a row out of the iterator - if (!iterator.hasNext()) - return endOfData(); - - Row current = iterator.next(); - DecoratedKey key = current.key; - - if (!range.stopKey().isMinimum() && range.stopKey().compareTo(key) < 0) - return endOfData(); - - // skipping outside of assigned range - if (!range.contains(key)) - continue; - - if (logger.isTraceEnabled()) - logger.trace("scanned {}", metadata.getKeyValidator().getString(key.getKey())); - - return current; - } - } - - public void close() throws IOException - { - iterator.close(); - } - }; - } - - @VisibleForTesting - public List getRangeSlice(final AbstractBounds range, - List rowFilter, - IDiskAtomFilter columnFilter, - int maxResults) - { - return getRangeSlice(range, rowFilter, columnFilter, maxResults, System.currentTimeMillis()); - } - - public List getRangeSlice(final AbstractBounds range, - List rowFilter, - IDiskAtomFilter columnFilter, - int maxResults, - long now) - { - return getRangeSlice(makeExtendedFilter(range, columnFilter, rowFilter, maxResults, false, false, now)); - } - - /** - * Allows generic range paging with the slice column filter. - * Typically, suppose we have rows A, B, C ... Z having each some columns in [1, 100]. - * And suppose we want to page through the query that for all rows returns the columns - * within [25, 75]. For that, we need to be able to do a range slice starting at (row r, column c) - * and ending at (row Z, column 75), *but* that only return columns in [25, 75]. - * That is what this method allows. The columnRange is the "window" of columns we are interested - * in each row, and columnStart (resp. columnEnd) is the start (resp. end) for the first - * (resp. last) requested row. - */ - public ExtendedFilter makeExtendedFilter(AbstractBounds keyRange, - SliceQueryFilter columnRange, - Composite columnStart, - Composite columnStop, - List rowFilter, - int maxResults, - boolean countCQL3Rows, - long now) - { - DataRange dataRange = new DataRange.Paging(keyRange, columnRange, columnStart, columnStop, metadata); - return ExtendedFilter.create(this, dataRange, rowFilter, maxResults, countCQL3Rows, now); - } - - public List getRangeSlice(AbstractBounds range, - List rowFilter, - IDiskAtomFilter columnFilter, - int maxResults, - long now, - boolean countCQL3Rows, - boolean isPaging) - { - return getRangeSlice(makeExtendedFilter(range, columnFilter, rowFilter, maxResults, countCQL3Rows, isPaging, now)); - } - - public ExtendedFilter makeExtendedFilter(AbstractBounds range, - IDiskAtomFilter columnFilter, - List rowFilter, - int maxResults, - boolean countCQL3Rows, - boolean isPaging, - long timestamp) - { - DataRange dataRange; - if (isPaging) - { - assert columnFilter instanceof SliceQueryFilter; - SliceQueryFilter sfilter = (SliceQueryFilter)columnFilter; - assert sfilter.slices.length == 1; - // create a new SliceQueryFilter that selects all cells, but pass the original slice start and finish - // through to DataRange.Paging to be used on the first and last partitions - SliceQueryFilter newFilter = new SliceQueryFilter(ColumnSlice.ALL_COLUMNS_ARRAY, sfilter.isReversed(), sfilter.count); - dataRange = new DataRange.Paging(range, newFilter, sfilter.start(), sfilter.finish(), metadata); - } - else - { - dataRange = new DataRange(range, columnFilter); - } - return ExtendedFilter.create(this, dataRange, rowFilter, maxResults, countCQL3Rows, timestamp); - } - - public List getRangeSlice(ExtendedFilter filter) - { - long start = System.nanoTime(); - try (OpOrder.Group op = readOrdering.start()) - { - return filter(getSequentialIterator(filter.dataRange, filter.timestamp), filter); - } - finally - { - metric.rangeLatency.addNano(System.nanoTime() - start); - } - } - - @VisibleForTesting - public List search(AbstractBounds range, - List clause, - IDiskAtomFilter dataFilter, - int maxResults) - { - return search(range, clause, dataFilter, maxResults, System.currentTimeMillis()); - } - - public List search(AbstractBounds range, - List clause, - IDiskAtomFilter dataFilter, - int maxResults, - long now) - { - return search(makeExtendedFilter(range, dataFilter, clause, maxResults, false, false, now)); - } - - public List search(ExtendedFilter filter) - { - Tracing.trace("Executing indexed scan for {}", filter.dataRange.keyRange().getString(metadata.getKeyValidator())); - return indexManager.search(filter); - } - - public List filter(AbstractScanIterator rowIterator, ExtendedFilter filter) - { - logger.trace("Filtering {} for rows matching {}", rowIterator, filter); - List rows = new ArrayList(); - int columnsCount = 0; - int total = 0, matched = 0; - boolean ignoreTombstonedPartitions = filter.ignoreTombstonedPartitions(); - - try - { - while (rowIterator.hasNext() && matched < filter.maxRows() && columnsCount < filter.maxColumns()) - { - // get the raw columns requested, and additional columns for the expressions if necessary - Row rawRow = rowIterator.next(); - total++; - ColumnFamily data = rawRow.cf; - - if (rowIterator.needsFiltering()) - { - IDiskAtomFilter extraFilter = filter.getExtraFilter(rawRow.key, data); - if (extraFilter != null) - { - ColumnFamily cf = filter.cfs.getColumnFamily(new QueryFilter(rawRow.key, name, extraFilter, filter.timestamp)); - if (cf != null) - data.addAll(cf); - } - - removeDroppedColumns(data); - - if (!filter.isSatisfiedBy(rawRow.key, data, null, null)) - continue; - - logger.trace("{} satisfies all filter expressions", data); - // cut the resultset back to what was requested, if necessary - data = filter.prune(rawRow.key, data); - } - else - { - removeDroppedColumns(data); - } - - rows.add(new Row(rawRow.key, data)); - if (!ignoreTombstonedPartitions || !data.hasOnlyTombstones(filter.timestamp)) - matched++; - - if (data != null) - columnsCount += filter.lastCounted(data); - // Update the underlying filter to avoid querying more columns per slice than necessary and to handle paging - filter.updateFilter(columnsCount); - } - - return rows; - } - finally - { - try - { - rowIterator.close(); - Tracing.trace("Scanned {} rows and matched {}", total, matched); - } - catch (IOException e) - { - throw new RuntimeException(e); - } - } - } - - public CellNameType getComparator() + public ClusteringComparator getComparator() { return metadata.comparator; } @@ -2388,20 +1817,20 @@ public class ColumnFamilyStore implements ColumnFamilyStoreMBean } /** - * @return the cached row for @param key if it is already present in the cache. - * That is, unlike getThroughCache, it will not readAndCache the row if it is not present, nor + * @return the cached partition for @param key if it is already present in the cache. + * Not that this will not readAndCache the parition if it is not present, nor * are these calls counted in cache statistics. * - * Note that this WILL cause deserialization of a SerializingCache row, so if all you - * need to know is whether a row is present or not, use containsCachedRow instead. + * Note that this WILL cause deserialization of a SerializingCache partition, so if all you + * need to know is whether a partition is present or not, use containsCachedParition instead. */ - public ColumnFamily getRawCachedRow(DecoratedKey key) + public CachedPartition getRawCachedPartition(DecoratedKey key) { if (!isRowCacheEnabled()) return null; IRowCacheEntry cached = CacheService.instance.rowCache.getInternal(new RowCacheKey(metadata.cfId, key)); - return cached == null || cached instanceof RowCacheSentinel ? null : (ColumnFamily)cached; + return cached == null || cached instanceof RowCacheSentinel ? null : (CachedPartition)cached; } private void invalidateCaches() @@ -2415,37 +1844,37 @@ public class ColumnFamilyStore implements ColumnFamilyStoreMBean /** * @return true if @param key is contained in the row cache */ - public boolean containsCachedRow(DecoratedKey key) + public boolean containsCachedParition(DecoratedKey key) { return CacheService.instance.rowCache.getCapacity() != 0 && CacheService.instance.rowCache.containsKey(new RowCacheKey(metadata.cfId, key)); } - public void invalidateCachedRow(RowCacheKey key) + public void invalidateCachedPartition(RowCacheKey key) { CacheService.instance.rowCache.remove(key); } - public void invalidateCachedRow(DecoratedKey key) + public void invalidateCachedPartition(DecoratedKey key) { UUID cfId = Schema.instance.getId(keyspace.getName(), this.name); if (cfId == null) return; // secondary index - invalidateCachedRow(new RowCacheKey(cfId, key)); + invalidateCachedPartition(new RowCacheKey(cfId, key)); } - public ClockAndCount getCachedCounter(ByteBuffer partitionKey, CellName cellName) + public ClockAndCount getCachedCounter(ByteBuffer partitionKey, Clustering clustering, ColumnDefinition column, CellPath path) { if (CacheService.instance.counterCache.getCapacity() == 0L) // counter cache disabled. return null; - return CacheService.instance.counterCache.get(CounterCacheKey.create(metadata.cfId, partitionKey, cellName)); + return CacheService.instance.counterCache.get(CounterCacheKey.create(metadata.cfId, partitionKey, clustering, column, path)); } - public void putCachedCounter(ByteBuffer partitionKey, CellName cellName, ClockAndCount clockAndCount) + public void putCachedCounter(ByteBuffer partitionKey, Clustering clustering, ColumnDefinition column, CellPath path, ClockAndCount clockAndCount) { if (CacheService.instance.counterCache.getCapacity() == 0L) // counter cache disabled. return; - CacheService.instance.counterCache.put(CounterCacheKey.create(metadata.cfId, partitionKey, cellName), clockAndCount); + CacheService.instance.counterCache.put(CounterCacheKey.create(metadata.cfId, partitionKey, clustering, column, path), clockAndCount); } public void forceMajorCompaction() throws InterruptedException, ExecutionException @@ -2830,7 +2259,7 @@ public class ColumnFamilyStore implements ColumnFamilyStoreMBean return view.sstables.isEmpty() && view.getCurrentMemtable().getOperations() == 0 && view.liveMemtables.size() <= 1 && view.flushingMemtables.size() == 0; } - private boolean isRowCacheEnabled() + public boolean isRowCacheEnabled() { return metadata.getCaching().rowCache.isEnabled() && CacheService.instance.rowCache.getCapacity() > 0; } diff --git a/src/java/org/apache/cassandra/db/ColumnIndex.java b/src/java/org/apache/cassandra/db/ColumnIndex.java index d9d6a9cf92..1a9b92d559 100644 --- a/src/java/org/apache/cassandra/db/ColumnIndex.java +++ b/src/java/org/apache/cassandra/db/ColumnIndex.java @@ -18,15 +18,15 @@ package org.apache.cassandra.db; import java.io.IOException; -import java.nio.ByteBuffer; import java.util.*; import com.google.common.annotations.VisibleForTesting; import org.apache.cassandra.config.DatabaseDescriptor; -import org.apache.cassandra.db.composites.Composite; +import org.apache.cassandra.db.rows.*; import org.apache.cassandra.io.sstable.IndexHelper; -import org.apache.cassandra.io.util.DataOutputPlus; +import org.apache.cassandra.io.sstable.format.Version; +import org.apache.cassandra.io.util.SequentialWriter; import org.apache.cassandra.utils.ByteBufferUtil; public class ColumnIndex @@ -42,6 +42,14 @@ public class ColumnIndex this.columnsIndex = columnsIndex; } + public static ColumnIndex writeAndBuildIndex(UnfilteredRowIterator iterator, SequentialWriter output, SerializationHeader header, Version version) throws IOException + { + assert !iterator.isEmpty() && version.storeRows(); + + Builder builder = new Builder(iterator, output, header, version.correspondingMessagingVersion()); + return builder.build(); + } + @VisibleForTesting public static ColumnIndex nothing() { @@ -52,192 +60,114 @@ public class ColumnIndex * Help to create an index for a column family based on size of columns, * and write said columns to disk. */ - public static class Builder + private static class Builder { + private final UnfilteredRowIterator iterator; + private final SequentialWriter writer; + private final SerializationHeader header; + private final int version; + private final ColumnIndex result; - private final long indexOffset; + private final long initialPosition; private long startPosition = -1; - private long endPosition = 0; - private long blockSize; - private OnDiskAtom firstColumn; - private OnDiskAtom lastColumn; - private OnDiskAtom lastBlockClosing; - private final DataOutputPlus output; - private final RangeTombstone.Tracker tombstoneTracker; - private int atomCount; - private final ByteBuffer key; - private final DeletionInfo deletionInfo; // only used for serializing and calculating row header size - private final OnDiskAtom.Serializer atomSerializer; + private int written; - public Builder(ColumnFamily cf, - ByteBuffer key, - DataOutputPlus output) + private ClusteringPrefix firstClustering; + private final ReusableClusteringPrefix lastClustering; + + private DeletionTime openMarker; + + public Builder(UnfilteredRowIterator iterator, + SequentialWriter writer, + SerializationHeader header, + int version) { - assert cf != null; - assert key != null; - assert output != null; + this.iterator = iterator; + this.writer = writer; + this.header = header; + this.version = version; - this.key = key; - deletionInfo = cf.deletionInfo(); - this.indexOffset = rowHeaderSize(key, deletionInfo); this.result = new ColumnIndex(new ArrayList()); - this.output = output; - this.tombstoneTracker = new RangeTombstone.Tracker(cf.getComparator()); - this.atomSerializer = cf.getComparator().onDiskAtomSerializer(); + this.initialPosition = writer.getFilePointer(); + this.lastClustering = new ReusableClusteringPrefix(iterator.metadata().clusteringColumns().size()); } - /** - * Returns the number of bytes between the beginning of the row and the - * first serialized column. - */ - private static long rowHeaderSize(ByteBuffer key, DeletionInfo delInfo) + private void writePartitionHeader(UnfilteredRowIterator iterator) throws IOException { - TypeSizes typeSizes = TypeSizes.NATIVE; - // TODO fix constantSize when changing the nativeconststs. - int keysize = key.remaining(); - return typeSizes.sizeof((short) keysize) + keysize // Row key - + DeletionTime.serializer.serializedSize(delInfo.getTopLevelDeletion(), typeSizes); + ByteBufferUtil.writeWithShortLength(iterator.partitionKey().getKey(), writer.stream); + DeletionTime.serializer.serialize(iterator.partitionLevelDeletion(), writer.stream); + if (header.hasStatic()) + UnfilteredSerializer.serializer.serialize(iterator.staticRow(), header, writer.stream, version); } - public RangeTombstone.Tracker tombstoneTracker() + public ColumnIndex build() throws IOException { - return tombstoneTracker; + writePartitionHeader(iterator); + + while (iterator.hasNext()) + add(iterator.next()); + + return close(); } - public int writtenAtomCount() + private long currentPosition() { - return atomCount + tombstoneTracker.writtenAtom(); + return writer.getFilePointer() - initialPosition; } - /** - * Serializes the index into in-memory structure with all required components - * such as Bloom Filter, index block size, IndexInfo list - * - * @param cf Column family to create index for - * - * @return information about index - it's Bloom Filter, block size and IndexInfo list - */ - public ColumnIndex build(ColumnFamily cf) throws IOException + private void addIndexBlock() { - // cf has disentangled the columns and range tombstones, we need to re-interleave them in comparator order - Comparator comparator = cf.getComparator(); - DeletionInfo.InOrderTester tester = cf.deletionInfo().inOrderTester(); - Iterator rangeIter = cf.deletionInfo().rangeIterator(); - RangeTombstone tombstone = rangeIter.hasNext() ? rangeIter.next() : null; + IndexHelper.IndexInfo cIndexInfo = new IndexHelper.IndexInfo(firstClustering, + lastClustering.get().takeAlias(), + startPosition, + currentPosition() - startPosition, + openMarker); + result.columnsIndex.add(cIndexInfo); + firstClustering = null; + } - for (Cell c : cf) + private void add(Unfiltered unfiltered) throws IOException + { + lastClustering.copy(unfiltered.clustering()); + boolean isMarker = unfiltered.kind() == Unfiltered.Kind.RANGE_TOMBSTONE_MARKER; + + if (firstClustering == null) { - while (tombstone != null && comparator.compare(c.name(), tombstone.min) >= 0) - { - // skip range tombstones that are shadowed by partition tombstones - if (!cf.deletionInfo().getTopLevelDeletion().isDeleted(tombstone)) - add(tombstone); - tombstone = rangeIter.hasNext() ? rangeIter.next() : null; - } - - // We can skip any cell if it's shadowed by a tombstone already. This is a more - // general case than was handled by CASSANDRA-2589. - if (!tester.isDeleted(c)) - add(c); + // Beginning of an index block. Remember the start and position + firstClustering = lastClustering.get().takeAlias(); + startPosition = currentPosition(); } - while (tombstone != null) + UnfilteredSerializer.serializer.serialize(unfiltered, header, writer.stream, version); + ++written; + + if (isMarker) { - add(tombstone); - tombstone = rangeIter.hasNext() ? rangeIter.next() : null; + RangeTombstoneMarker marker = (RangeTombstoneMarker) unfiltered; + openMarker = marker.isOpen(false) ? marker.openDeletionTime(false) : null; } - ColumnIndex index = build(); - - maybeWriteEmptyRowHeader(); - - return index; - } - - /** - * The important distinction wrt build() is that we may be building for a row that ends up - * being compacted away entirely, i.e., the input consists only of expired tombstones (or - * columns shadowed by expired tombstone). Thus, it is the caller's responsibility - * to decide whether to write the header for an empty row. - */ - public ColumnIndex buildForCompaction(Iterator columns) throws IOException - { - while (columns.hasNext()) - { - OnDiskAtom c = columns.next(); - add(c); - } - - return build(); - } - - public void add(OnDiskAtom column) throws IOException - { - atomCount++; - - if (firstColumn == null) - { - firstColumn = column; - startPosition = endPosition; - // TODO: have that use the firstColumn as min + make sure we optimize that on read - endPosition += tombstoneTracker.writeOpenedMarker(firstColumn, output, atomSerializer); - blockSize = 0; // We don't count repeated tombstone marker in the block size, to avoid a situation - // where we wouldn't make any progress because a block is filled by said marker - } - - long size = atomSerializer.serializedSizeForSSTable(column); - endPosition += size; - blockSize += size; // if we hit the column index size that we have to index after, go ahead and index it. - if (blockSize >= DatabaseDescriptor.getColumnIndexSize()) - { - IndexHelper.IndexInfo cIndexInfo = new IndexHelper.IndexInfo(firstColumn.name(), column.name(), indexOffset + startPosition, endPosition - startPosition); - result.columnsIndex.add(cIndexInfo); - firstColumn = null; - lastBlockClosing = column; - } - - maybeWriteRowHeader(); - atomSerializer.serializeForSSTable(column, output); - - // TODO: Should deal with removing unneeded tombstones - tombstoneTracker.update(column, false); - - lastColumn = column; + if (currentPosition() - startPosition >= DatabaseDescriptor.getColumnIndexSize()) + addIndexBlock(); } - private void maybeWriteRowHeader() throws IOException + private ColumnIndex close() throws IOException { - if (lastColumn == null) - { - ByteBufferUtil.writeWithShortLength(key, output); - DeletionTime.serializer.serialize(deletionInfo.getTopLevelDeletion(), output); - } - } + UnfilteredSerializer.serializer.writeEndOfPartition(writer.stream); - public ColumnIndex build() - { - // all columns were GC'd after all - if (lastColumn == null) + // It's possible we add no rows, just a top level deletion + if (written == 0) return ColumnIndex.EMPTY; // the last column may have fallen on an index boundary already. if not, index it explicitly. - if (result.columnsIndex.isEmpty() || lastBlockClosing != lastColumn) - { - IndexHelper.IndexInfo cIndexInfo = new IndexHelper.IndexInfo(firstColumn.name(), lastColumn.name(), indexOffset + startPosition, endPosition - startPosition); - result.columnsIndex.add(cIndexInfo); - } + if (firstClustering != null) + addIndexBlock(); // we should always have at least one computed index block, but we only write it out if there is more than that. assert result.columnsIndex.size() > 0; return result; } - - public void maybeWriteEmptyRowHeader() throws IOException - { - if (!deletionInfo.isLive()) - maybeWriteRowHeader(); - } } } diff --git a/src/java/org/apache/cassandra/db/ColumnSerializer.java b/src/java/org/apache/cassandra/db/ColumnSerializer.java deleted file mode 100644 index 8e7026caa2..0000000000 --- a/src/java/org/apache/cassandra/db/ColumnSerializer.java +++ /dev/null @@ -1,188 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.apache.cassandra.db; - -import java.io.DataInput; -import java.io.IOException; -import java.nio.ByteBuffer; - -import org.apache.cassandra.db.composites.CellName; -import org.apache.cassandra.db.composites.CellNameType; -import org.apache.cassandra.io.ISerializer; -import org.apache.cassandra.io.FSReadError; -import org.apache.cassandra.io.util.DataOutputPlus; -import org.apache.cassandra.io.util.FileDataInput; -import org.apache.cassandra.io.util.FileUtils; -import org.apache.cassandra.utils.ByteBufferUtil; - -public class ColumnSerializer implements ISerializer -{ - public final static int DELETION_MASK = 0x01; - public final static int EXPIRATION_MASK = 0x02; - public final static int COUNTER_MASK = 0x04; - public final static int COUNTER_UPDATE_MASK = 0x08; - public final static int RANGE_TOMBSTONE_MASK = 0x10; - - /** - * Flag affecting deserialization behavior. - * - LOCAL: for deserialization of local data (Expired columns are - * converted to tombstones (to gain disk space)). - * - FROM_REMOTE: for deserialization of data received from remote hosts - * (Expired columns are converted to tombstone and counters have - * their delta cleared) - * - PRESERVE_SIZE: used when no transformation must be performed, i.e, - * when we must ensure that deserializing and reserializing the - * result yield the exact same bytes. Streaming uses this. - */ - public static enum Flag - { - LOCAL, FROM_REMOTE, PRESERVE_SIZE; - } - - private final CellNameType type; - - public ColumnSerializer(CellNameType type) - { - this.type = type; - } - - public void serialize(Cell cell, DataOutputPlus out) throws IOException - { - assert !cell.name().isEmpty(); - type.cellSerializer().serialize(cell.name(), out); - try - { - out.writeByte(cell.serializationFlags()); - if (cell instanceof CounterCell) - { - out.writeLong(((CounterCell) cell).timestampOfLastDelete()); - } - else if (cell instanceof ExpiringCell) - { - out.writeInt(((ExpiringCell) cell).getTimeToLive()); - out.writeInt(cell.getLocalDeletionTime()); - } - out.writeLong(cell.timestamp()); - ByteBufferUtil.writeWithLength(cell.value(), out); - } - catch (IOException e) - { - throw new RuntimeException(e); - } - } - - public Cell deserialize(DataInput in) throws IOException - { - return deserialize(in, Flag.LOCAL); - } - - /* - * For counter columns, we must know when we deserialize them if what we - * deserialize comes from a remote host. If it does, then we must clear - * the delta. - */ - public Cell deserialize(DataInput in, ColumnSerializer.Flag flag) throws IOException - { - return deserialize(in, flag, Integer.MIN_VALUE); - } - - public Cell deserialize(DataInput in, ColumnSerializer.Flag flag, int expireBefore) throws IOException - { - CellName name = type.cellSerializer().deserialize(in); - - int b = in.readUnsignedByte(); - return deserializeColumnBody(in, name, b, flag, expireBefore); - } - - Cell deserializeColumnBody(DataInput in, CellName name, int mask, ColumnSerializer.Flag flag, int expireBefore) throws IOException - { - if ((mask & COUNTER_MASK) != 0) - { - long timestampOfLastDelete = in.readLong(); - long ts = in.readLong(); - ByteBuffer value = ByteBufferUtil.readWithLength(in); - return BufferCounterCell.create(name, value, ts, timestampOfLastDelete, flag); - } - else if ((mask & EXPIRATION_MASK) != 0) - { - int ttl = in.readInt(); - int expiration = in.readInt(); - long ts = in.readLong(); - ByteBuffer value = ByteBufferUtil.readWithLength(in); - return BufferExpiringCell.create(name, value, ts, ttl, expiration, expireBefore, flag); - } - else - { - long ts = in.readLong(); - ByteBuffer value = ByteBufferUtil.readWithLength(in); - return (mask & COUNTER_UPDATE_MASK) != 0 - ? new BufferCounterUpdateCell(name, value, ts) - : ((mask & DELETION_MASK) == 0 - ? new BufferCell(name, value, ts) - : new BufferDeletedCell(name, value, ts)); - } - } - - void skipColumnBody(DataInput in, int mask) throws IOException - { - if ((mask & COUNTER_MASK) != 0) - FileUtils.skipBytesFully(in, 16); - else if ((mask & EXPIRATION_MASK) != 0) - FileUtils.skipBytesFully(in, 16); - else - FileUtils.skipBytesFully(in, 8); - - int length = in.readInt(); - FileUtils.skipBytesFully(in, length); - } - - public long serializedSize(Cell cell, TypeSizes typeSizes) - { - return cell.serializedSize(type, typeSizes); - } - - public static class CorruptColumnException extends IOException - { - public CorruptColumnException(String s) - { - super(s); - } - - public static CorruptColumnException create(DataInput in, ByteBuffer name) - { - assert name.remaining() <= 0; - String format = "invalid column name length %d%s"; - String details = ""; - if (in instanceof FileDataInput) - { - FileDataInput fdis = (FileDataInput)in; - long remaining; - try - { - remaining = fdis.bytesRemaining(); - } - catch (IOException e) - { - throw new FSReadError(e, fdis.getPath()); - } - details = String.format(" (%s, %d bytes remaining)", fdis.getPath(), remaining); - } - return new CorruptColumnException(String.format(format, name.remaining(), details)); - } - } -} diff --git a/src/java/org/apache/cassandra/db/Columns.java b/src/java/org/apache/cassandra/db/Columns.java new file mode 100644 index 0000000000..83d39db9a5 --- /dev/null +++ b/src/java/org/apache/cassandra/db/Columns.java @@ -0,0 +1,535 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.cassandra.db; + +import java.io.DataInput; +import java.io.IOException; +import java.util.*; +import java.nio.ByteBuffer; +import java.security.MessageDigest; + +import com.google.common.collect.AbstractIterator; +import com.google.common.collect.Iterators; + +import org.apache.cassandra.config.CFMetaData; +import org.apache.cassandra.config.ColumnDefinition; +import org.apache.cassandra.db.marshal.UTF8Type; +import org.apache.cassandra.db.marshal.MapType; +import org.apache.cassandra.io.util.DataOutputPlus; +import org.apache.cassandra.utils.ByteBufferUtil; + +/** + * An immutable and sorted list of (non-PK) columns for a given table. + *

+ * Note that in practice, it will either store only static columns, or only regular ones. When + * we need both type of columns, we use a {@link PartitionColumns} object. + */ +public class Columns implements Iterable +{ + public static final Serializer serializer = new Serializer(); + public static final Columns NONE = new Columns(new ColumnDefinition[0], 0); + + public final ColumnDefinition[] columns; + public final int complexIdx; // Index of the first complex column + + private Columns(ColumnDefinition[] columns, int complexIdx) + { + assert complexIdx <= columns.length; + this.columns = columns; + this.complexIdx = complexIdx; + } + + private Columns(ColumnDefinition[] columns) + { + this(columns, findFirstComplexIdx(columns)); + } + + /** + * Creates a {@code Columns} holding only the one column provided. + * + * @param c the column for which to create a {@code Columns} object. + * + * @return the newly created {@code Columns} containing only {@code c}. + */ + public static Columns of(ColumnDefinition c) + { + ColumnDefinition[] columns = new ColumnDefinition[]{ c }; + return new Columns(columns, c.isComplex() ? 0 : 1); + } + + /** + * Returns a new {@code Columns} object holing the same columns than the provided set. + * + * @param param s the set from which to create the new {@code Columns}. + * + * @return the newly created {@code Columns} containing the columns from {@code s}. + */ + public static Columns from(Set s) + { + ColumnDefinition[] columns = s.toArray(new ColumnDefinition[s.size()]); + Arrays.sort(columns); + return new Columns(columns, findFirstComplexIdx(columns)); + } + + private static int findFirstComplexIdx(ColumnDefinition[] columns) + { + for (int i = 0; i < columns.length; i++) + if (columns[i].isComplex()) + return i; + return columns.length; + } + + /** + * Whether this columns is empty. + * + * @return whether this columns is empty. + */ + public boolean isEmpty() + { + return columns.length == 0; + } + + /** + * The number of simple columns in this object. + * + * @return the number of simple columns in this object. + */ + public int simpleColumnCount() + { + return complexIdx; + } + + /** + * The number of complex columns (non-frozen collections, udts, ...) in this object. + * + * @return the number of complex columns in this object. + */ + public int complexColumnCount() + { + return columns.length - complexIdx; + } + + /** + * The total number of columns in this object. + * + * @return the total number of columns in this object. + */ + public int columnCount() + { + return columns.length; + } + + /** + * Whether this objects contains simple columns. + * + * @return whether this objects contains simple columns. + */ + public boolean hasSimple() + { + return complexIdx > 0; + } + + /** + * Whether this objects contains complex columns. + * + * @return whether this objects contains complex columns. + */ + public boolean hasComplex() + { + return complexIdx < columns.length; + } + + /** + * Returns the ith simple column of this object. + * + * @param i the index for the simple column to fectch. This must + * satisfy {@code 0 <= i < simpleColumnCount()}. + * + * @return the {@code i}th simple column in this object. + */ + public ColumnDefinition getSimple(int i) + { + return columns[i]; + } + + /** + * Returns the ith complex column of this object. + * + * @param i the index for the complex column to fectch. This must + * satisfy {@code 0 <= i < complexColumnCount()}. + * + * @return the {@code i}th complex column in this object. + */ + public ColumnDefinition getComplex(int i) + { + return columns[complexIdx + i]; + } + + /** + * The index of the provided simple column in this object (if it contains + * the provided column). + * + * @param c the simple column for which to return the index of. + * @param from the index to start the search from. + * + * @return the index for simple column {@code c} if it is contains in this + * object (starting from index {@code from}), {@code -1} otherwise. + */ + public int simpleIdx(ColumnDefinition c, int from) + { + assert !c.isComplex(); + for (int i = from; i < complexIdx; i++) + // We know we only use "interned" ColumnIdentifier so == is ok. + if (columns[i].name == c.name) + return i; + return -1; + } + + /** + * The index of the provided complex column in this object (if it contains + * the provided column). + * + * @param c the complex column for which to return the index of. + * @param from the index to start the search from. + * + * @return the index for complex column {@code c} if it is contains in this + * object (starting from index {@code from}), {@code -1} otherwise. + */ + public int complexIdx(ColumnDefinition c, int from) + { + assert c.isComplex(); + for (int i = complexIdx + from; i < columns.length; i++) + // We know we only use "interned" ColumnIdentifier so == is ok. + if (columns[i].name == c.name) + return i - complexIdx; + return -1; + } + + /** + * Whether the provided column is contained by this object. + * + * @param c the column to check presence of. + * + * @return whether {@code c} is contained by this object. + */ + public boolean contains(ColumnDefinition c) + { + return c.isComplex() ? complexIdx(c, 0) >= 0 : simpleIdx(c, 0) >= 0; + } + + /** + * Whether or not there is some counter columns within those columns. + * + * @return whether or not there is some counter columns within those columns. + */ + public boolean hasCounters() + { + for (int i = 0; i < complexIdx; i++) + { + if (columns[i].type.isCounter()) + return true; + } + + for (int i = complexIdx; i < columns.length; i++) + { + // We only support counter in maps because that's all we need for now (and we need it for the sake of thrift super columns of counter) + if (columns[i].type instanceof MapType && (((MapType)columns[i].type).valueComparator().isCounter())) + return true; + } + + return false; + } + + /** + * Returns the result of merging this {@code Columns} object with the + * provided one. + * + * @param other the other {@code Columns} to merge this object with. + * + * @return the result of merging/taking the union of {@code this} and + * {@code other}. The returned object may be one of the operand and that + * operand is a subset of the other operand. + */ + public Columns mergeTo(Columns other) + { + if (this == other || other == NONE) + return this; + if (this == NONE) + return other; + + int i = 0, j = 0; + int size = 0; + while (i < columns.length && j < other.columns.length) + { + ++size; + int cmp = columns[i].compareTo(other.columns[j]); + if (cmp == 0) + { + ++i; + ++j; + } + else if (cmp < 0) + { + ++i; + } + else + { + ++j; + } + } + + // If every element was always counted on both array, we have the same + // arrays for the first min elements + if (i == size && j == size) + { + // We've exited because of either c1 or c2 (or both). The array that + // made us stop is thus a subset of the 2nd one, return that array. + return i == columns.length ? other : this; + } + + size += i == columns.length ? other.columns.length - j : columns.length - i; + ColumnDefinition[] result = new ColumnDefinition[size]; + i = 0; + j = 0; + for (int k = 0; k < size; k++) + { + int cmp = i >= columns.length ? 1 + : (j >= other.columns.length ? -1 : columns[i].compareTo(other.columns[j])); + if (cmp == 0) + { + result[k] = columns[i]; + ++i; + ++j; + } + else if (cmp < 0) + { + result[k] = columns[i++]; + } + else + { + result[k] = other.columns[j++]; + } + } + return new Columns(result, findFirstComplexIdx(result)); + } + + /** + * Whether this object is a subset of the provided other {@code Columns object}. + * + * @param other the othere object to test for inclusion in this object. + * + * @return whether all the columns of {@code other} are contained by this object. + */ + public boolean contains(Columns other) + { + if (other.columns.length > columns.length) + return false; + + int j = 0; + int cmp = 0; + for (ColumnDefinition def : other.columns) + { + while (j < columns.length && (cmp = columns[j].compareTo(def)) < 0) + j++; + + if (j >= columns.length || cmp > 0) + return false; + + // cmp == 0, we've found the definition. Ce can bump j once more since + // we know we won't need to compare that element again + j++; + } + return true; + } + + /** + * Iterator over the simple columns of this object. + * + * @return an iterator over the simple columns of this object. + */ + public Iterator simpleColumns() + { + return new ColumnIterator(0, complexIdx); + } + + /** + * Iterator over the complex columns of this object. + * + * @return an iterator over the complex columns of this object. + */ + public Iterator complexColumns() + { + return new ColumnIterator(complexIdx, columns.length); + } + + /** + * Iterator over all the columns of this object. + * + * @return an iterator over all the columns of this object. + */ + public Iterator iterator() + { + return Iterators.forArray(columns); + } + + /** + * An iterator that returns the columns of this object in "select" order (that + * is in global alphabetical order, where the "normal" iterator returns simple + * columns first and the complex second). + * + * @return an iterator returning columns in alphabetical order. + */ + public Iterator selectOrderIterator() + { + // In wildcard selection, we want to return all columns in alphabetical order, + // irregarding of whether they are complex or not + return new AbstractIterator() + { + private int regular; + private int complex = complexIdx; + + protected ColumnDefinition computeNext() + { + if (complex >= columns.length) + return regular >= complexIdx ? endOfData() : columns[regular++]; + if (regular >= complexIdx) + return columns[complex++]; + + return ByteBufferUtil.compareUnsigned(columns[regular].name.bytes, columns[complex].name.bytes) < 0 + ? columns[regular++] + : columns[complex++]; + } + }; + } + + /** + * Returns the equivalent of those columns but with the provided column removed. + * + * @param column the column to remove. + * + * @return newly allocated columns containing all the columns of {@code this} expect + * for {@code column}. + */ + public Columns without(ColumnDefinition column) + { + int idx = column.isComplex() ? complexIdx(column, 0) : simpleIdx(column, 0); + if (idx < 0) + return this; + + int realIdx = column.isComplex() ? complexIdx + idx : idx; + + ColumnDefinition[] newColumns = new ColumnDefinition[columns.length - 1]; + System.arraycopy(columns, 0, newColumns, 0, realIdx); + System.arraycopy(columns, realIdx + 1, newColumns, realIdx, newColumns.length - realIdx); + return new Columns(newColumns); + } + + public void digest(MessageDigest digest) + { + for (ColumnDefinition c : this) + digest.update(c.name.bytes.duplicate()); + } + + @Override + public boolean equals(Object other) + { + if (!(other instanceof Columns)) + return false; + + Columns that = (Columns)other; + return this.complexIdx == that.complexIdx && Arrays.equals(this.columns, that.columns); + } + + @Override + public int hashCode() + { + return Objects.hash(complexIdx, Arrays.hashCode(columns)); + } + + @Override + public String toString() + { + StringBuilder sb = new StringBuilder(); + boolean first = true; + for (ColumnDefinition def : this) + { + if (first) first = false; else sb.append(" "); + sb.append(def.name); + } + return sb.toString(); + } + + private class ColumnIterator extends AbstractIterator + { + private final int to; + private int idx; + + private ColumnIterator(int from, int to) + { + this.idx = from; + this.to = to; + } + + protected ColumnDefinition computeNext() + { + if (idx >= to) + return endOfData(); + return columns[idx++]; + } + } + + public static class Serializer + { + public void serialize(Columns columns, DataOutputPlus out) throws IOException + { + out.writeShort(columns.columnCount()); + for (ColumnDefinition column : columns) + ByteBufferUtil.writeWithShortLength(column.name.bytes, out); + } + + public long serializedSize(Columns columns, TypeSizes sizes) + { + long size = sizes.sizeof((short)columns.columnCount()); + for (ColumnDefinition column : columns) + size += sizes.sizeofWithShortLength(column.name.bytes); + return size; + } + + public Columns deserialize(DataInput in, CFMetaData metadata) throws IOException + { + int length = in.readUnsignedShort(); + ColumnDefinition[] columns = new ColumnDefinition[length]; + for (int i = 0; i < length; i++) + { + ByteBuffer name = ByteBufferUtil.readWithShortLength(in); + ColumnDefinition column = metadata.getColumnDefinition(name); + if (column == null) + { + // If we don't find the definition, it could be we have data for a dropped column, and we shouldn't + // fail deserialization because of that. So we grab a "fake" ColumnDefinition that ensure proper + // deserialization. The column will be ignore later on anyway. + column = metadata.getDroppedColumnDefinition(name); + if (column == null) + throw new RuntimeException("Unknown column " + UTF8Type.instance.getString(name) + " during deserialization"); + } + columns[i] = column; + } + return new Columns(columns); + } + } +} diff --git a/src/java/org/apache/cassandra/db/CompactTables.java b/src/java/org/apache/cassandra/db/CompactTables.java new file mode 100644 index 0000000000..a72e7f23eb --- /dev/null +++ b/src/java/org/apache/cassandra/db/CompactTables.java @@ -0,0 +1,176 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.cassandra.db; + +import java.nio.ByteBuffer; +import java.util.*; + +import org.apache.cassandra.config.CFMetaData; +import org.apache.cassandra.config.ColumnDefinition; +import org.apache.cassandra.db.marshal.*; +import org.apache.cassandra.utils.ByteBufferUtil; + +/** + * Small utility methods pertaining to the encoding of COMPACT STORAGE tables. + * + * COMPACT STORAGE tables exists mainly for the sake of encoding internally thrift tables (as well as + * exposing those tables through CQL). Note that due to these constraints, the internal representation + * of compact tables does *not* correspond exactly to their CQL definition. + * + * The internal layout of such tables is such that it can encode any thrift table. That layout is as follow: + * CREATE TABLE compact ( + * key [key_validation_class], + * [column_metadata_1] [type1] static, + * ..., + * [column_metadata_n] [type1] static, + * column [comparator], + * value [default_validation_class] + * PRIMARY KEY (key, column) + * ) + * More specifically, the table: + * - always has a clustering column and a regular value, which are used to store the "dynamic" thrift columns name and value. + * Those are always present because we have no way to know in advance if "dynamic" columns will be inserted or not. Note + * that when declared from CQL, compact tables may not have any clustering: in that case, we still have a clustering + * defined internally, it is just ignored as far as interacting from CQL is concerned. + * - have a static column for every "static" column defined in the thrift "column_metadata". Note that when declaring a compact + * table from CQL without any clustering (but some non-PK columns), the columns ends up static internally even though they are + * not in the declaration + * + * On variation is that if the table comparator is a CompositeType, then the underlying table will have one clustering column by + * element of the CompositeType, but the rest of the layout is as above. + * + * As far as thrift is concerned, one exception to this is super column families, which have a different layout. Namely, a super + * column families is encoded with: + * CREATE TABLE super ( + * key [key_validation_class], + * super_column_name [comparator], + * [column_metadata_1] [type1], + * ..., + * [column_metadata_n] [type1], + * "" map<[sub_comparator], [default_validation_class]> + * PRIMARY KEY (key, super_column_name) + * ) + * In other words, every super column is encoded by a row. That row has one column for each defined "column_metadata", but it also + * has a special map column (whose name is the empty string as this is guaranteed to never conflict with a user-defined + * "column_metadata") which stores the super column "dynamic" sub-columns. + */ +public abstract class CompactTables +{ + // We use an empty value for the 1) this can't conflict with a user-defined column and 2) this actually + // validate with any comparator which makes it convenient for columnDefinitionComparator(). + public static final ByteBuffer SUPER_COLUMN_MAP_COLUMN = ByteBufferUtil.EMPTY_BYTE_BUFFER; + public static final String SUPER_COLUMN_MAP_COLUMN_STR = UTF8Type.instance.compose(SUPER_COLUMN_MAP_COLUMN); + + private CompactTables() {} + + public static ColumnDefinition getCompactValueColumn(PartitionColumns columns, boolean isSuper) + { + if (isSuper) + { + for (ColumnDefinition column : columns.regulars) + if (column.name.bytes.equals(SUPER_COLUMN_MAP_COLUMN)) + return column; + throw new AssertionError("Invalid super column table definition, no 'dynamic' map column"); + } + assert columns.regulars.simpleColumnCount() == 1 && columns.regulars.complexColumnCount() == 0; + return columns.regulars.getSimple(0); + } + + public static AbstractType columnDefinitionComparator(ColumnDefinition.Kind kind, boolean isSuper, AbstractType rawComparator, AbstractType rawSubComparator) + { + if (isSuper) + return kind == ColumnDefinition.Kind.REGULAR ? rawSubComparator : UTF8Type.instance; + else + return kind == ColumnDefinition.Kind.STATIC ? rawComparator : UTF8Type.instance; + } + + public static boolean hasEmptyCompactValue(CFMetaData metadata) + { + return metadata.compactValueColumn().type instanceof EmptyType; + } + + public static boolean isSuperColumnMapColumn(ColumnDefinition column) + { + return column.kind == ColumnDefinition.Kind.REGULAR && column.name.bytes.equals(SUPER_COLUMN_MAP_COLUMN); + } + + public static DefaultNames defaultNameGenerator(Set usedNames) + { + return new DefaultNames(new HashSet(usedNames)); + } + + public static DefaultNames defaultNameGenerator(Iterable defs) + { + Set usedNames = new HashSet<>(); + for (ColumnDefinition def : defs) + usedNames.add(def.name.toString()); + return new DefaultNames(usedNames); + } + + public static class DefaultNames + { + private static final String DEFAULT_PARTITION_KEY_NAME = "key"; + private static final String DEFAULT_CLUSTERING_NAME = "column"; + private static final String DEFAULT_COMPACT_VALUE_NAME = "value"; + + private final Set usedNames; + private int partitionIndex = 0; + private int clusteringIndex = 1; + private int compactIndex = 0; + + private DefaultNames(Set usedNames) + { + this.usedNames = usedNames; + } + + public String defaultPartitionKeyName() + { + while (true) + { + // For compatibility sake, we call the first alias 'key' rather than 'key1'. This + // is inconsistent with column alias, but it's probably not worth risking breaking compatibility now. + String candidate = partitionIndex == 0 ? DEFAULT_PARTITION_KEY_NAME : DEFAULT_PARTITION_KEY_NAME + (partitionIndex + 1); + ++partitionIndex; + if (usedNames.add(candidate)) + return candidate; + } + } + + public String defaultClusteringName() + { + while (true) + { + String candidate = DEFAULT_CLUSTERING_NAME + clusteringIndex; + ++clusteringIndex; + if (usedNames.add(candidate)) + return candidate; + } + } + + public String defaultCompactValueName() + { + while (true) + { + String candidate = compactIndex == 0 ? DEFAULT_COMPACT_VALUE_NAME : DEFAULT_COMPACT_VALUE_NAME + compactIndex; + ++compactIndex; + if (usedNames.add(candidate)) + return candidate; + } + } + } +} diff --git a/src/java/org/apache/cassandra/db/Conflicts.java b/src/java/org/apache/cassandra/db/Conflicts.java new file mode 100644 index 0000000000..fa0e819e82 --- /dev/null +++ b/src/java/org/apache/cassandra/db/Conflicts.java @@ -0,0 +1,79 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.cassandra.db; + +import java.nio.ByteBuffer; + +import org.apache.cassandra.db.context.CounterContext; + +public abstract class Conflicts +{ + private Conflicts() {} + + public enum Resolution { LEFT_WINS, MERGE, RIGHT_WINS }; + + public static Resolution resolveRegular(long leftTimestamp, + boolean leftLive, + int leftLocalDeletionTime, + ByteBuffer leftValue, + long rightTimestamp, + boolean rightLive, + int rightLocalDeletionTime, + ByteBuffer rightValue) + { + if (leftTimestamp != rightTimestamp) + return leftTimestamp < rightTimestamp ? Resolution.RIGHT_WINS : Resolution.LEFT_WINS; + + if (leftLive != rightLive) + return leftLive ? Resolution.RIGHT_WINS : Resolution.LEFT_WINS; + + int c = leftValue.compareTo(rightValue); + if (c < 0) + return Resolution.RIGHT_WINS; + else if (c > 0) + return Resolution.LEFT_WINS; + + // Prefer the longest ttl if relevant + return leftLocalDeletionTime < rightLocalDeletionTime ? Resolution.RIGHT_WINS : Resolution.LEFT_WINS; + } + + public static Resolution resolveCounter(long leftTimestamp, + boolean leftLive, + ByteBuffer leftValue, + long rightTimestamp, + boolean rightLive, + ByteBuffer rightValue) + { + // No matter what the counter cell's timestamp is, a tombstone always takes precedence. See CASSANDRA-7346. + if (!leftLive) + // left is a tombstone: it has precedence over right if either right is not a tombstone, or left has a greater timestamp + return rightLive || leftTimestamp > rightTimestamp ? Resolution.LEFT_WINS : Resolution.RIGHT_WINS; + + // If right is a tombstone, since left isn't one, it has precedence + if (!rightLive) + return Resolution.RIGHT_WINS; + + return Resolution.MERGE; + } + + public static ByteBuffer mergeCounterValues(ByteBuffer left, ByteBuffer right) + { + return CounterContext.instance().merge(left, right); + } + +} diff --git a/src/java/org/apache/cassandra/db/CounterCell.java b/src/java/org/apache/cassandra/db/CounterCell.java deleted file mode 100644 index cda1200ca2..0000000000 --- a/src/java/org/apache/cassandra/db/CounterCell.java +++ /dev/null @@ -1,44 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.apache.cassandra.db; - -import org.apache.cassandra.config.CFMetaData; -import org.apache.cassandra.db.context.CounterContext; -import org.apache.cassandra.utils.concurrent.OpOrder; -import org.apache.cassandra.utils.memory.AbstractAllocator; -import org.apache.cassandra.utils.memory.MemtableAllocator; - -/** - * A column that represents a partitioned counter. - */ -public interface CounterCell extends Cell -{ - static final CounterContext contextManager = CounterContext.instance(); - - public long timestampOfLastDelete(); - - public long total(); - - public boolean hasLegacyShards(); - - public Cell markLocalToBeCleared(); - - CounterCell localCopy(CFMetaData metadata, AbstractAllocator allocator); - - CounterCell localCopy(CFMetaData metaData, MemtableAllocator allocator, OpOrder.Group opGroup); -} diff --git a/src/java/org/apache/cassandra/db/CounterMutation.java b/src/java/org/apache/cassandra/db/CounterMutation.java index 58717b4b24..f87c66c445 100644 --- a/src/java/org/apache/cassandra/db/CounterMutation.java +++ b/src/java/org/apache/cassandra/db/CounterMutation.java @@ -19,7 +19,6 @@ package org.apache.cassandra.db; import java.io.DataInput; import java.io.IOException; -import java.nio.ByteBuffer; import java.util.*; import java.util.concurrent.TimeUnit; import java.util.concurrent.locks.Lock; @@ -27,12 +26,15 @@ import java.util.concurrent.locks.Lock; import com.google.common.base.Function; import com.google.common.base.Objects; import com.google.common.collect.Iterables; +import com.google.common.collect.Iterators; +import com.google.common.collect.PeekingIterator; import com.google.common.util.concurrent.Striped; import org.apache.cassandra.config.DatabaseDescriptor; -import org.apache.cassandra.db.composites.CellName; +import org.apache.cassandra.db.rows.*; +import org.apache.cassandra.db.filter.*; +import org.apache.cassandra.db.partitions.*; import org.apache.cassandra.db.context.CounterContext; -import org.apache.cassandra.db.filter.NamesQueryFilter; import org.apache.cassandra.exceptions.WriteTimeoutException; import org.apache.cassandra.io.IVersionedSerializer; import org.apache.cassandra.io.util.DataOutputPlus; @@ -41,6 +43,7 @@ import org.apache.cassandra.net.MessagingService; import org.apache.cassandra.service.CacheService; import org.apache.cassandra.tracing.Tracing; import org.apache.cassandra.utils.*; +import org.apache.cassandra.utils.concurrent.OpOrder; public class CounterMutation implements IMutation { @@ -67,9 +70,9 @@ public class CounterMutation implements IMutation return mutation.getColumnFamilyIds(); } - public Collection getColumnFamilies() + public Collection getPartitionUpdates() { - return mutation.getColumnFamilies(); + return mutation.getPartitionUpdates(); } public Mutation getMutation() @@ -77,7 +80,7 @@ public class CounterMutation implements IMutation return mutation; } - public ByteBuffer key() + public DecoratedKey key() { return mutation.key(); } @@ -111,19 +114,14 @@ public class CounterMutation implements IMutation Mutation result = new Mutation(getKeyspaceName(), key()); Keyspace keyspace = Keyspace.open(getKeyspaceName()); - int count = 0; - for (ColumnFamily cf : getColumnFamilies()) - count += cf.getColumnCount(); - - List locks = new ArrayList<>(count); - Tracing.trace("Acquiring {} counter locks", count); + List locks = new ArrayList<>(); + Tracing.trace("Acquiring counter locks"); try { grabCounterLocks(keyspace, locks); - for (ColumnFamily cf : getColumnFamilies()) - result.add(processModifications(cf)); + for (PartitionUpdate upd : getPartitionUpdates()) + result.add(processModifications(upd)); result.apply(); - updateCounterCache(result, keyspace); return result; } finally @@ -160,141 +158,144 @@ public class CounterMutation implements IMutation */ private Iterable getCounterLockKeys() { - return Iterables.concat(Iterables.transform(getColumnFamilies(), new Function>() + return Iterables.concat(Iterables.transform(getPartitionUpdates(), new Function>() { - public Iterable apply(final ColumnFamily cf) + public Iterable apply(final PartitionUpdate update) { - return Iterables.transform(cf, new Function() + return Iterables.concat(Iterables.transform(update, new Function>() { - public Object apply(Cell cell) + public Iterable apply(final Row row) { - return Objects.hashCode(cf.id(), key(), cell.name()); + return Iterables.concat(Iterables.transform(row, new Function() + { + public Object apply(final Cell cell) + { + return Objects.hashCode(update.metadata().cfId, key(), row.clustering(), cell.column(), cell.path()); + } + })); } - }); + })); } })); } - // Replaces all the CounterUpdateCell-s with updated regular CounterCell-s - private ColumnFamily processModifications(ColumnFamily changesCF) + private PartitionUpdate processModifications(PartitionUpdate changes) { - ColumnFamilyStore cfs = Keyspace.open(getKeyspaceName()).getColumnFamilyStore(changesCF.id()); + ColumnFamilyStore cfs = Keyspace.open(getKeyspaceName()).getColumnFamilyStore(changes.metadata().cfId); - ColumnFamily resultCF = changesCF.cloneMeShallow(); - - List counterUpdateCells = new ArrayList<>(changesCF.getColumnCount()); - for (Cell cell : changesCF) - { - if (cell instanceof CounterUpdateCell) - counterUpdateCells.add((CounterUpdateCell)cell); - else - resultCF.addColumn(cell); - } - - if (counterUpdateCells.isEmpty()) - return resultCF; // only DELETEs - - ClockAndCount[] currentValues = getCurrentValues(counterUpdateCells, cfs); - for (int i = 0; i < counterUpdateCells.size(); i++) - { - ClockAndCount currentValue = currentValues[i]; - CounterUpdateCell update = counterUpdateCells.get(i); - - long clock = currentValue.clock + 1L; - long count = currentValue.count + update.delta(); - - resultCF.addColumn(new BufferCounterCell(update.name(), - CounterContext.instance().createGlobal(CounterId.getLocalId(), clock, count), - update.timestamp())); - } - - return resultCF; - } - - // Attempt to load the current values(s) from cache. If that fails, read the rest from the cfs. - private ClockAndCount[] getCurrentValues(List counterUpdateCells, ColumnFamilyStore cfs) - { - ClockAndCount[] currentValues = new ClockAndCount[counterUpdateCells.size()]; - int remaining = counterUpdateCells.size(); + List marks = changes.collectCounterMarks(); if (CacheService.instance.counterCache.getCapacity() != 0) { - Tracing.trace("Fetching {} counter values from cache", counterUpdateCells.size()); - remaining = getCurrentValuesFromCache(counterUpdateCells, cfs, currentValues); - if (remaining == 0) - return currentValues; + Tracing.trace("Fetching {} counter values from cache", marks.size()); + updateWithCurrentValuesFromCache(marks, cfs); + if (marks.isEmpty()) + return changes; } - Tracing.trace("Reading {} counter values from the CF", remaining); - getCurrentValuesFromCFS(counterUpdateCells, cfs, currentValues); + Tracing.trace("Reading {} counter values from the CF", marks.size()); + updateWithCurrentValuesFromCFS(marks, cfs); - return currentValues; + // What's remain is new counters + for (PartitionUpdate.CounterMark mark : marks) + updateWithCurrentValue(mark, ClockAndCount.BLANK, cfs); + + return changes; + } + + private void updateWithCurrentValue(PartitionUpdate.CounterMark mark, ClockAndCount currentValue, ColumnFamilyStore cfs) + { + long clock = currentValue.clock + 1L; + long count = currentValue.count + CounterContext.instance().total(mark.value()); + + mark.setValue(CounterContext.instance().createGlobal(CounterId.getLocalId(), clock, count)); + + // Cache the newly updated value + cfs.putCachedCounter(key().getKey(), mark.clustering(), mark.column(), mark.path(), ClockAndCount.create(clock, count)); } // Returns the count of cache misses. - private int getCurrentValuesFromCache(List counterUpdateCells, - ColumnFamilyStore cfs, - ClockAndCount[] currentValues) + private void updateWithCurrentValuesFromCache(List marks, ColumnFamilyStore cfs) { - int cacheMisses = 0; - for (int i = 0; i < counterUpdateCells.size(); i++) + Iterator iter = marks.iterator(); + while (iter.hasNext()) { - ClockAndCount cached = cfs.getCachedCounter(key(), counterUpdateCells.get(i).name()); + PartitionUpdate.CounterMark mark = iter.next(); + ClockAndCount cached = cfs.getCachedCounter(key().getKey(), mark.clustering(), mark.column(), mark.path()); if (cached != null) - currentValues[i] = cached; - else - cacheMisses++; + { + updateWithCurrentValue(mark, cached, cfs); + iter.remove(); + } } - return cacheMisses; } // Reads the missing current values from the CFS. - private void getCurrentValuesFromCFS(List counterUpdateCells, - ColumnFamilyStore cfs, - ClockAndCount[] currentValues) + private void updateWithCurrentValuesFromCFS(List marks, ColumnFamilyStore cfs) { - SortedSet names = new TreeSet<>(cfs.metadata.comparator); - for (int i = 0; i < currentValues.length; i++) - if (currentValues[i] == null) - names.add(counterUpdateCells.get(i).name()); - - ReadCommand cmd = new SliceByNamesReadCommand(getKeyspaceName(), key(), cfs.metadata.cfName, Long.MIN_VALUE, new NamesQueryFilter(names)); - Row row = cmd.getRow(cfs.keyspace); - ColumnFamily cf = row == null ? null : row.cf; - - for (int i = 0; i < currentValues.length; i++) + ColumnFilter.Builder builder = ColumnFilter.selectionBuilder(); + NavigableSet names = new TreeSet<>(cfs.metadata.comparator); + for (PartitionUpdate.CounterMark mark : marks) { - if (currentValues[i] != null) - continue; - - Cell cell = cf == null ? null : cf.getColumn(counterUpdateCells.get(i).name()); - if (cell == null || !cell.isLive()) // absent or a tombstone. - currentValues[i] = ClockAndCount.BLANK; + names.add(mark.clustering().takeAlias()); + if (mark.path() == null) + builder.add(mark.column()); else - currentValues[i] = CounterContext.instance().getLocalClockAndCount(cell.value()); + builder.select(mark.column(), mark.path()); + } + + int nowInSec = FBUtilities.nowInSeconds(); + ClusteringIndexNamesFilter filter = new ClusteringIndexNamesFilter(names, false); + SinglePartitionReadCommand cmd = SinglePartitionReadCommand.create(cfs.metadata, nowInSec, key(), builder.build(), filter); + PeekingIterator markIter = Iterators.peekingIterator(marks.iterator()); + try (OpOrder.Group op = cfs.readOrdering.start(); RowIterator partition = UnfilteredRowIterators.filter(cmd.queryMemtableAndDisk(cfs, op), nowInSec)) + { + updateForRow(markIter, partition.staticRow(), cfs); + + while (partition.hasNext()) + { + if (!markIter.hasNext()) + return; + + updateForRow(markIter, partition.next(), cfs); + } } } - private void updateCounterCache(Mutation applied, Keyspace keyspace) + private int compare(Clustering c1, Clustering c2, ColumnFamilyStore cfs) { - if (CacheService.instance.counterCache.getCapacity() == 0) + if (c1 == Clustering.STATIC_CLUSTERING) + return c2 == Clustering.STATIC_CLUSTERING ? 0 : -1; + if (c2 == Clustering.STATIC_CLUSTERING) + return 1; + + return cfs.getComparator().compare(c1, c2); + } + + private void updateForRow(PeekingIterator markIter, Row row, ColumnFamilyStore cfs) + { + int cmp = 0; + // If the mark is before the row, we have no value for this mark, just consume it + while (markIter.hasNext() && (cmp = compare(markIter.peek().clustering(), row.clustering(), cfs)) < 0) + markIter.next(); + + if (!markIter.hasNext()) return; - for (ColumnFamily cf : applied.getColumnFamilies()) + while (cmp == 0) { - ColumnFamilyStore cfs = keyspace.getColumnFamilyStore(cf.id()); - for (Cell cell : cf) - if (cell instanceof CounterCell) - cfs.putCachedCounter(key(), cell.name(), CounterContext.instance().getLocalClockAndCount(cell.value())); - } - } + PartitionUpdate.CounterMark mark = markIter.next(); + Cell cell = mark.path() == null ? row.getCell(mark.column()) : row.getCell(mark.column(), mark.path()); + if (cell != null) + { + updateWithCurrentValue(mark, CounterContext.instance().getLocalClockAndCount(cell.value()), cfs); + markIter.remove(); + } + if (!markIter.hasNext()) + return; - public void addAll(IMutation m) - { - if (!(m instanceof CounterMutation)) - throw new IllegalArgumentException(); - CounterMutation cm = (CounterMutation)m; - mutation.addAll(cm.mutation); + cmp = compare(markIter.peek().clustering(), row.clustering(), cfs); + } } public long getTimeout() diff --git a/src/java/org/apache/cassandra/db/DataRange.java b/src/java/org/apache/cassandra/db/DataRange.java index 1e6f8c8ab8..909d6ed00c 100644 --- a/src/java/org/apache/cassandra/db/DataRange.java +++ b/src/java/org/apache/cassandra/db/DataRange.java @@ -6,7 +6,6 @@ * to you under the Apache License, Version 2.0 (the * "License"); you may not use this file except in compliance * with the License. You may obtain a copy of the License at - * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software @@ -17,302 +16,391 @@ */ package org.apache.cassandra.db; +import java.io.DataInput; +import java.io.IOException; import java.nio.ByteBuffer; -import java.util.ArrayList; -import java.util.Comparator; -import java.util.List; -import com.google.common.base.Objects; import org.apache.cassandra.config.CFMetaData; -import org.apache.cassandra.db.columniterator.IdentityQueryFilter; -import org.apache.cassandra.db.composites.Composite; -import org.apache.cassandra.db.composites.Composites; +import org.apache.cassandra.config.ColumnDefinition; import org.apache.cassandra.db.filter.*; +import org.apache.cassandra.db.marshal.AbstractType; +import org.apache.cassandra.db.marshal.CompositeType; import org.apache.cassandra.dht.*; +import org.apache.cassandra.io.util.DataOutputPlus; +import org.apache.cassandra.net.MessagingService; /** - * Groups key range and column filter for range queries. - * - * The main "trick" of this class is that the column filter can only - * be obtained by providing the row key on which the column filter will - * be applied (which we always know before actually querying the columns). - * - * This allows the paging DataRange to return a filter for most rows but a - * potentially different ones for the starting and stopping key. Could - * allow more fancy stuff in the future too, like column filters that - * depend on the actual key value :) + * Groups both the range of partitions to query, and the clustering index filter to + * apply for each partition (for a (partition) range query). + *

+ * The main "trick" is that the clustering index filter can only be obtained by + * providing the partition key on which the filter will be applied. This is + * necessary when paging range queries, as we might need a different filter + * for the starting key than for other keys (because the previous page we had + * queried may have ended in the middle of a partition). */ public class DataRange { - protected final AbstractBounds keyRange; - protected IDiskAtomFilter columnFilter; - protected final boolean selectFullRow; + public static final Serializer serializer = new Serializer(); - public DataRange(AbstractBounds range, IDiskAtomFilter columnFilter) + private final AbstractBounds keyRange; + protected final ClusteringIndexFilter clusteringIndexFilter; + + /** + * Creates a {@code DataRange} given a range of partition keys and a clustering index filter. The + * return {@code DataRange} will return the same filter for all keys. + * + * @param range the range over partition keys to use. + * @param clusteringIndexFilter the clustering index filter to use. + */ + public DataRange(AbstractBounds range, ClusteringIndexFilter clusteringIndexFilter) { this.keyRange = range; - this.columnFilter = columnFilter; - this.selectFullRow = columnFilter instanceof SliceQueryFilter - ? isFullRowSlice((SliceQueryFilter)columnFilter) - : false; - } - - public static boolean isFullRowSlice(SliceQueryFilter filter) - { - return filter.slices.length == 1 - && filter.start().isEmpty() - && filter.finish().isEmpty() - && filter.count == Integer.MAX_VALUE; + this.clusteringIndexFilter = clusteringIndexFilter; } + /** + * Creates a {@code DataRange} to query all data (over the whole ring). + * + * @param partitioner the partitioner in use for the table. + * + * @return the newly create {@code DataRange}. + */ public static DataRange allData(IPartitioner partitioner) { return forTokenRange(new Range(partitioner.getMinimumToken(), partitioner.getMinimumToken())); } - public static DataRange forTokenRange(Range keyRange) + /** + * Creates a {@code DataRange} to query all rows over the provided token range. + * + * @param tokenRange the (partition key) token range to query. + * + * @return the newly create {@code DataRange}. + */ + public static DataRange forTokenRange(Range tokenRange) { - return forKeyRange(Range.makeRowRange(keyRange)); + return forKeyRange(Range.makeRowRange(tokenRange)); } - public static DataRange forKeyRange(Range keyRange) + /** + * Creates a {@code DataRange} to query all rows over the provided key range. + * + * @param keyRange the (partition key) range to query. + * + * @return the newly create {@code DataRange}. + */ + public static DataRange forKeyRange(Range keyRange) { - return new DataRange(keyRange, new IdentityQueryFilter()); + return new DataRange(keyRange, new ClusteringIndexSliceFilter(Slices.ALL, false)); } - public AbstractBounds keyRange() + /** + * Creates a {@code DataRange} to query all partitions of the ring using the provided + * clustering index filter. + * + * @param partitioner the partitioner in use for the table queried. + * @param filter the clustering index filter to use. + * + * @return the newly create {@code DataRange}. + */ + public static DataRange allData(IPartitioner partitioner, ClusteringIndexFilter filter) + { + return new DataRange(Range.makeRowRange(new Range(partitioner.getMinimumToken(), partitioner.getMinimumToken())), filter); + } + + /** + * The range of partition key queried by this {@code DataRange}. + * + * @return the range of partition key queried by this {@code DataRange}. + */ + public AbstractBounds keyRange() { return keyRange; } - public RowPosition startKey() + /** + * The start of the partition key range queried by this {@code DataRange}. + * + * @return the start of the partition key range queried by this {@code DataRange}. + */ + public PartitionPosition startKey() { return keyRange.left; } - public RowPosition stopKey() + /** + * The end of the partition key range queried by this {@code DataRange}. + * + * @return the end of the partition key range queried by this {@code DataRange}. + */ + public PartitionPosition stopKey() { return keyRange.right; } /** - * Returns true if tombstoned partitions should not be included in results or count towards the limit. - * See CASSANDRA-8490 for more details on why this is needed (and done this way). - * */ - public boolean ignoredTombstonedPartitions() + * Whether the underlying clustering index filter is a names filter or not. + * + * @return Whether the underlying clustering index filter is a names filter or not. + */ + public boolean isNamesQuery() { - if (!(columnFilter instanceof SliceQueryFilter)) - return false; - - return ((SliceQueryFilter) columnFilter).compositesToGroup == SliceQueryFilter.IGNORE_TOMBSTONED_PARTITIONS; + return clusteringIndexFilter instanceof ClusteringIndexNamesFilter; } - // Whether the bounds of this DataRange actually wraps around. + /** + * Whether the range queried by this {@code DataRange} actually wraps around. + * + * @return whether the range queried by this {@code DataRange} actually wraps around. + */ public boolean isWrapAround() { - // On range can ever wrap + // Only range can ever wrap return keyRange instanceof Range && ((Range)keyRange).isWrapAround(); } - public boolean contains(RowPosition pos) + /** + * Whether the provided ring position is covered by this {@code DataRange}. + * + * @return whether the provided ring position is covered by this {@code DataRange}. + */ + public boolean contains(PartitionPosition pos) { return keyRange.contains(pos); } - public int getLiveCount(ColumnFamily data, long now) + /** + * Whether this {@code DataRange} queries everything (has no restriction neither on the + * partition queried, nor within the queried partition). + * + * @return Whether this {@code DataRange} queries everything. + */ + public boolean isUnrestricted() { - return columnFilter instanceof SliceQueryFilter - ? ((SliceQueryFilter)columnFilter).lastCounted() - : columnFilter.getLiveCount(data, now); - } - - public boolean selectsFullRowFor(ByteBuffer rowKey) - { - return selectFullRow; + return startKey().isMinimum() && stopKey().isMinimum() && clusteringIndexFilter.selectsAllPartition(); } /** - * Returns a column filter that should be used for a particular row key. Note that in the case of paging, - * slice starts and ends may change depending on the row key. + * The clustering index filter to use for the provided key. + *

+ * This may or may not be the same filter for all keys (that is, paging range + * use a different filter for their start key). + * + * @param key the partition key for which we want the clustering index filter. + * + * @return the clustering filter to use for {@code key}. */ - public IDiskAtomFilter columnFilter(ByteBuffer rowKey) + public ClusteringIndexFilter clusteringIndexFilter(DecoratedKey key) { - return columnFilter; + return clusteringIndexFilter; } /** - * Sets a new limit on the number of (grouped) cells to fetch. This is currently only used when the query limit applies - * to CQL3 rows. + * Returns a new {@code DataRange} for use when paging {@code this} range. + * + * @param range the range of partition keys to query. + * @param comparator the comparator for the table queried. + * @param lastReturned the clustering for the last result returned by the previous page, i.e. the result we want to start our new page + * from. This last returned must must correspond to left bound of {@code range} (in other words, {@code range.left} must be the + * partition key for that {@code lastReturned} result). + * @param inclusive whether or not we want to include the {@code lastReturned} in the newly returned page of results. + * + * @return a new {@code DataRange} suitable for paging {@code this} range given the {@code lastRetuned} result of the previous page. */ - public void updateColumnsLimit(int count) + public DataRange forPaging(AbstractBounds range, ClusteringComparator comparator, Clustering lastReturned, boolean inclusive) { - columnFilter.updateColumnsLimit(count); + return new Paging(range, clusteringIndexFilter, comparator, lastReturned, inclusive); } - public static class Paging extends DataRange + /** + * Returns a new {@code DataRange} equivalent to {@code this} one but restricted to the provided sub-range. + * + * @param range the sub-range to use for the newly returned data range. Note that assumes that {@code range} is a proper + * sub-range of the initial range but doesn't validate it. You should make sure to only provided sub-ranges however or this + * might throw off the paging case (see Paging.forSubRange()). + * + * @return a new {@code DataRange} using {@code range} as partition key range and the clustering index filter filter from {@code this}. + */ + public DataRange forSubRange(AbstractBounds range) { - // The slice of columns that we want to fetch for each row, ignoring page start/end issues. - private final SliceQueryFilter sliceFilter; + return new DataRange(range, clusteringIndexFilter); + } - private final CFMetaData cfm; + public String toString(CFMetaData metadata) + { + return String.format("range=%s pfilter=%s", keyRange.getString(metadata.getKeyValidator()), clusteringIndexFilter.toString(metadata)); + } - private final Comparator comparator; + public String toCQLString(CFMetaData metadata) + { + if (isUnrestricted()) + return "UNRESTRICTED"; - // used to restrict the start of the slice for the first partition in the range - private final Composite firstPartitionColumnStart; + StringBuilder sb = new StringBuilder(); - // used to restrict the end of the slice for the last partition in the range - private final Composite lastPartitionColumnFinish; + boolean needAnd = false; + if (!startKey().isMinimum()) + { + appendClause(startKey(), sb, metadata, true, keyRange.isStartInclusive()); + needAnd = true; + } + if (!stopKey().isMinimum()) + { + if (needAnd) + sb.append(" AND "); + appendClause(stopKey(), sb, metadata, false, keyRange.isEndInclusive()); + needAnd = true; + } - // tracks the last key that we updated the filter for to avoid duplicating work - private ByteBuffer lastKeyFilterWasUpdatedFor; + String filterString = clusteringIndexFilter.toCQLString(metadata); + if (!filterString.isEmpty()) + sb.append(needAnd ? " AND " : "").append(filterString); - private Paging(AbstractBounds range, SliceQueryFilter filter, Composite firstPartitionColumnStart, - Composite lastPartitionColumnFinish, CFMetaData cfm, Comparator comparator) + return sb.toString(); + } + + private void appendClause(PartitionPosition pos, StringBuilder sb, CFMetaData metadata, boolean isStart, boolean isInclusive) + { + sb.append("token("); + sb.append(ColumnDefinition.toCQLString(metadata.partitionKeyColumns())); + sb.append(") ").append(getOperator(isStart, isInclusive)).append(" "); + if (pos instanceof DecoratedKey) + { + sb.append("token("); + appendKeyString(sb, metadata.getKeyValidator(), ((DecoratedKey)pos).getKey()); + sb.append(")"); + } + else + { + sb.append(((Token.KeyBound)pos).getToken()); + } + } + + private static String getOperator(boolean isStart, boolean isInclusive) + { + return isStart + ? (isInclusive ? ">=" : ">") + : (isInclusive ? "<=" : "<"); + } + + // TODO: this is reused in SinglePartitionReadCommand but this should not really be here. Ideally + // we need a more "native" handling of composite partition keys. + public static void appendKeyString(StringBuilder sb, AbstractType type, ByteBuffer key) + { + if (type instanceof CompositeType) + { + CompositeType ct = (CompositeType)type; + ByteBuffer[] values = ct.split(key); + for (int i = 0; i < ct.types.size(); i++) + sb.append(i == 0 ? "" : ", ").append(ct.types.get(i).getString(values[i])); + } + else + { + sb.append(type.getString(key)); + } + } + + /** + * Specialized {@code DataRange} used for the paging case. + *

+ * It uses the clustering of the last result of the previous page to restrict the filter on the + * first queried partition (the one for that last result) so it only fetch results that follow that + * last result. In other words, this makes sure this resume paging where we left off. + */ + private static class Paging extends DataRange + { + private final ClusteringComparator comparator; + private final Clustering lastReturned; + private final boolean inclusive; + + private Paging(AbstractBounds range, + ClusteringIndexFilter filter, + ClusteringComparator comparator, + Clustering lastReturned, + boolean inclusive) { super(range, filter); // When using a paging range, we don't allow wrapped ranges, as it's unclear how to handle them properly. - // This is ok for now since we only need this in range slice queries, and the range are "unwrapped" in that case. + // This is ok for now since we only need this in range queries, and the range are "unwrapped" in that case. assert !(range instanceof Range) || !((Range)range).isWrapAround() || range.right.isMinimum() : range; + assert lastReturned != null; - this.sliceFilter = filter; - this.cfm = cfm; this.comparator = comparator; - this.firstPartitionColumnStart = firstPartitionColumnStart; - this.lastPartitionColumnFinish = lastPartitionColumnFinish; - this.lastKeyFilterWasUpdatedFor = null; - } - - public Paging(AbstractBounds range, SliceQueryFilter filter, Composite columnStart, Composite columnFinish, CFMetaData cfm) - { - this(range, filter, columnStart, columnFinish, cfm, filter.isReversed() ? cfm.comparator.reverseComparator() : cfm.comparator); + this.lastReturned = lastReturned; + this.inclusive = inclusive; } @Override - public boolean selectsFullRowFor(ByteBuffer rowKey) + public ClusteringIndexFilter clusteringIndexFilter(DecoratedKey key) { - // If we initial filter is not the full filter, don't bother - if (!selectFullRow) - return false; - - if (!equals(startKey(), rowKey) && !equals(stopKey(), rowKey)) - return true; - - return isFullRowSlice((SliceQueryFilter)columnFilter(rowKey)); - } - - private boolean equals(RowPosition pos, ByteBuffer rowKey) - { - return pos instanceof DecoratedKey && ((DecoratedKey)pos).getKey().equals(rowKey); + return key.equals(startKey()) + ? clusteringIndexFilter.forPaging(comparator, lastReturned, inclusive) + : clusteringIndexFilter; } @Override - public IDiskAtomFilter columnFilter(ByteBuffer rowKey) + public DataRange forSubRange(AbstractBounds range) { - /* - * We have that ugly hack that for slice queries, when we ask for - * the live count, we reach into the query filter to get the last - * counter number of columns to avoid recounting. - * Maybe we should just remove that hack, but in the meantime, we - * need to keep a reference the last returned filter. - */ - if (equals(startKey(), rowKey) || equals(stopKey(), rowKey)) + // This is called for subrange of the initial range. So either it's the beginning of the initial range, + // and we need to preserver lastReturned, or it's not, and we don't care about it anymore. + return range.left.equals(keyRange().left) + ? new Paging(range, clusteringIndexFilter, comparator, lastReturned, inclusive) + : new DataRange(range, clusteringIndexFilter); + } + + @Override + public boolean isUnrestricted() + { + return false; + } + } + + public static class Serializer + { + public void serialize(DataRange range, DataOutputPlus out, int version, CFMetaData metadata) throws IOException + { + AbstractBounds.rowPositionSerializer.serialize(range.keyRange, out, version); + ClusteringIndexFilter.serializer.serialize(range.clusteringIndexFilter, out, version); + boolean isPaging = range instanceof Paging; + out.writeBoolean(isPaging); + if (isPaging) { - if (!rowKey.equals(lastKeyFilterWasUpdatedFor)) - { - this.lastKeyFilterWasUpdatedFor = rowKey; - columnFilter = sliceFilter.withUpdatedSlices(slicesForKey(rowKey)); - } + Clustering.serializer.serialize(((Paging)range).lastReturned, out, version, metadata.comparator.subtypes()); + out.writeBoolean(((Paging)range).inclusive); + } + } + + public DataRange deserialize(DataInput in, int version, CFMetaData metadata) throws IOException + { + AbstractBounds range = AbstractBounds.rowPositionSerializer.deserialize(in, MessagingService.globalPartitioner(), version); + ClusteringIndexFilter filter = ClusteringIndexFilter.serializer.deserialize(in, version, metadata); + if (in.readBoolean()) + { + ClusteringComparator comparator = metadata.comparator; + Clustering lastReturned = Clustering.serializer.deserialize(in, version, comparator.subtypes()); + boolean inclusive = in.readBoolean(); + return new Paging(range, filter, comparator, lastReturned, inclusive); } else { - columnFilter = sliceFilter; + return new DataRange(range, filter); } - - return columnFilter; } - /** Returns true if the slice includes static columns, false otherwise. */ - private boolean sliceIncludesStatics(ColumnSlice slice, boolean reversed, CFMetaData cfm) + public long serializedSize(DataRange range, int version, CFMetaData metadata) { - return cfm.hasStaticColumns() && - slice.includes(reversed ? cfm.comparator.reverseComparator() : cfm.comparator, cfm.comparator.staticPrefix().end()); - } + long size = AbstractBounds.rowPositionSerializer.serializedSize(range.keyRange, version) + + ClusteringIndexFilter.serializer.serializedSize(range.clusteringIndexFilter, version) + + 1; // isPaging boolean - private ColumnSlice[] slicesForKey(ByteBuffer key) - { - // Also note that firstPartitionColumnStart and lastPartitionColumnFinish, when used, only "restrict" the filter slices, - // it doesn't expand on them. As such, we can ignore the case where they are empty and we do - // as it screw up with the logic below (see #6592) - Composite newStart = equals(startKey(), key) && !firstPartitionColumnStart.isEmpty() ? firstPartitionColumnStart : null; - Composite newFinish = equals(stopKey(), key) && !lastPartitionColumnFinish.isEmpty() ? lastPartitionColumnFinish : null; - - // in the common case, we'll have the same number of slices - List newSlices = new ArrayList<>(sliceFilter.slices.length); - - // Check our slices to see if any fall before the page start (in which case they can be removed) or - // if they contain the page start (in which case they should start from the page start). However, if the - // slices would include static columns, we need to ensure they are also fetched, and so a separate - // slice for the static columns may be required. - // Note that if the query is reversed, we can't handle statics by simply adding a separate slice here, so - // the reversed case is handled by SliceFromReadCommand instead. See CASSANDRA-8502 for more details. - for (ColumnSlice slice : sliceFilter.slices) + if (range instanceof Paging) { - if (newStart != null) - { - if (slice.isBefore(comparator, newStart)) - { - if (!sliceFilter.reversed && sliceIncludesStatics(slice, false, cfm)) - newSlices.add(new ColumnSlice(Composites.EMPTY, cfm.comparator.staticPrefix().end())); - - continue; - } - - if (slice.includes(comparator, newStart)) - { - if (!sliceFilter.reversed && sliceIncludesStatics(slice, false, cfm) && !newStart.equals(Composites.EMPTY)) - newSlices.add(new ColumnSlice(Composites.EMPTY, cfm.comparator.staticPrefix().end())); - - slice = new ColumnSlice(newStart, slice.finish); - } - - // once we see a slice that either includes the page start or is after it, we can stop checking - // against the page start (because the slices are ordered) - newStart = null; - } - - assert newStart == null; - if (newFinish != null && !slice.isBefore(comparator, newFinish)) - { - if (slice.includes(comparator, newFinish)) - newSlices.add(new ColumnSlice(slice.start, newFinish)); - // In any case, we're done - break; - } - newSlices.add(slice); + size += Clustering.serializer.serializedSize(((Paging)range).lastReturned, version, metadata.comparator.subtypes(), TypeSizes.NATIVE); + size += 1; // inclusive boolean } - - return newSlices.toArray(new ColumnSlice[newSlices.size()]); - } - - @Override - public void updateColumnsLimit(int count) - { - columnFilter.updateColumnsLimit(count); - sliceFilter.updateColumnsLimit(count); - } - - @Override - public String toString() - { - return Objects.toStringHelper(this) - .add("keyRange", keyRange) - .add("sliceFilter", sliceFilter) - .add("columnFilter", columnFilter) - .add("firstPartitionColumnStart", firstPartitionColumnStart == null ? "null" : cfm.comparator.getString(firstPartitionColumnStart)) - .add("lastPartitionColumnFinish", lastPartitionColumnFinish == null ? "null" : cfm.comparator.getString(lastPartitionColumnFinish)) - .toString(); + return size; } } } diff --git a/src/java/org/apache/cassandra/db/DecoratedKey.java b/src/java/org/apache/cassandra/db/DecoratedKey.java index cc62a1594d..92d641460e 100644 --- a/src/java/org/apache/cassandra/db/DecoratedKey.java +++ b/src/java/org/apache/cassandra/db/DecoratedKey.java @@ -36,7 +36,7 @@ import org.apache.cassandra.utils.IFilter.FilterKey; * if this matters, you can subclass RP to use a stronger hash, or use a non-lossy tokenization scheme (as in the * OrderPreservingPartitioner classes). */ -public abstract class DecoratedKey implements RowPosition, FilterKey +public abstract class DecoratedKey implements PartitionPosition, FilterKey { public static final Comparator comparator = new Comparator() { @@ -72,7 +72,7 @@ public abstract class DecoratedKey implements RowPosition, FilterKey return ByteBufferUtil.compareUnsigned(getKey(), other.getKey()) == 0; // we compare faster than BB.equals for array backed BB } - public int compareTo(RowPosition pos) + public int compareTo(PartitionPosition pos) { if (this == pos) return 0; @@ -86,7 +86,7 @@ public abstract class DecoratedKey implements RowPosition, FilterKey return cmp == 0 ? ByteBufferUtil.compareUnsigned(getKey(), otherKey.getKey()) : cmp; } - public static int compareTo(IPartitioner partitioner, ByteBuffer key, RowPosition position) + public static int compareTo(IPartitioner partitioner, ByteBuffer key, PartitionPosition position) { // delegate to Token.KeyBound if needed if (!(position instanceof DecoratedKey)) @@ -113,9 +113,9 @@ public abstract class DecoratedKey implements RowPosition, FilterKey return false; } - public RowPosition.Kind kind() + public PartitionPosition.Kind kind() { - return RowPosition.Kind.ROW_KEY; + return PartitionPosition.Kind.ROW_KEY; } @Override diff --git a/src/java/org/apache/cassandra/db/DeletedCell.java b/src/java/org/apache/cassandra/db/DeletedCell.java deleted file mode 100644 index 998c4097af..0000000000 --- a/src/java/org/apache/cassandra/db/DeletedCell.java +++ /dev/null @@ -1,30 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.apache.cassandra.db; - -import org.apache.cassandra.config.CFMetaData; -import org.apache.cassandra.utils.concurrent.OpOrder; -import org.apache.cassandra.utils.memory.AbstractAllocator; -import org.apache.cassandra.utils.memory.MemtableAllocator; - -public interface DeletedCell extends Cell -{ - DeletedCell localCopy(CFMetaData metadata, AbstractAllocator allocator); - - DeletedCell localCopy(CFMetaData metaData, MemtableAllocator allocator, OpOrder.Group opGroup); -} diff --git a/src/java/org/apache/cassandra/db/DeletionInfo.java b/src/java/org/apache/cassandra/db/DeletionInfo.java index 048324a1f0..e54d6b1cf3 100644 --- a/src/java/org/apache/cassandra/db/DeletionInfo.java +++ b/src/java/org/apache/cassandra/db/DeletionInfo.java @@ -17,40 +17,32 @@ */ package org.apache.cassandra.db; -import java.io.DataInput; -import java.io.IOException; -import java.security.MessageDigest; -import java.util.Comparator; import java.util.Iterator; import com.google.common.base.Objects; import com.google.common.collect.Iterators; import org.apache.cassandra.cache.IMeasurableMemory; -import org.apache.cassandra.db.composites.CType; -import org.apache.cassandra.db.composites.Composite; -import org.apache.cassandra.io.IVersionedSerializer; -import org.apache.cassandra.io.util.DataOutputPlus; -import org.apache.cassandra.utils.ByteBufferUtil; +import org.apache.cassandra.db.rows.Cell; import org.apache.cassandra.utils.ObjectSizes; import org.apache.cassandra.utils.memory.AbstractAllocator; /** - * A combination of a top-level (or row) tombstone and range tombstones describing the deletions - * within a {@link ColumnFamily} (or row). + * A combination of a top-level (partition) tombstone and range tombstones describing the deletions + * within a partition. */ public class DeletionInfo implements IMeasurableMemory { private static final long EMPTY_SIZE = ObjectSizes.measure(new DeletionInfo(0, 0)); /** - * This represents a deletion of the entire row. We can't represent this within the RangeTombstoneList, so it's - * kept separately. This also slightly optimizes the common case of a full row deletion. + * This represents a deletion of the entire partition. We can't represent this within the RangeTombstoneList, so it's + * kept separately. This also slightly optimizes the common case of a full partition deletion. */ - private DeletionTime topLevel; + private DeletionTime partitionDeletion; /** - * A list of range tombstones within the row. This is left as null if there are no range tombstones + * A list of range tombstones within the partition. This is left as null if there are no range tombstones * (to save an allocation (since it's a common case). */ private RangeTombstoneList ranges; @@ -65,28 +57,23 @@ public class DeletionInfo implements IMeasurableMemory { // Pre-1.1 node may return MIN_VALUE for non-deleted container, but the new default is MAX_VALUE // (see CASSANDRA-3872) - this(new DeletionTime(markedForDeleteAt, localDeletionTime == Integer.MIN_VALUE ? Integer.MAX_VALUE : localDeletionTime)); + this(new SimpleDeletionTime(markedForDeleteAt, localDeletionTime == Integer.MIN_VALUE ? Integer.MAX_VALUE : localDeletionTime)); } - public DeletionInfo(DeletionTime topLevel) + public DeletionInfo(DeletionTime partitionDeletion) { - this(topLevel, null); + this(partitionDeletion, null); } - public DeletionInfo(Composite start, Composite end, Comparator comparator, long markedForDeleteAt, int localDeletionTime) + public DeletionInfo(ClusteringComparator comparator, Slice slice, long markedForDeleteAt, int localDeletionTime) { this(DeletionTime.LIVE, new RangeTombstoneList(comparator, 1)); - ranges.add(start, end, markedForDeleteAt, localDeletionTime); + ranges.add(slice.start(), slice.end(), markedForDeleteAt, localDeletionTime); } - public DeletionInfo(RangeTombstone rangeTombstone, Comparator comparator) + public DeletionInfo(DeletionTime partitionDeletion, RangeTombstoneList ranges) { - this(rangeTombstone.min, rangeTombstone.max, comparator, rangeTombstone.data.markedForDeleteAt, rangeTombstone.data.localDeletionTime); - } - - private DeletionInfo(DeletionTime topLevel, RangeTombstoneList ranges) - { - this.topLevel = topLevel; + this.partitionDeletion = partitionDeletion.takeAlias(); this.ranges = ranges; } @@ -100,17 +87,16 @@ public class DeletionInfo implements IMeasurableMemory public DeletionInfo copy() { - return new DeletionInfo(topLevel, ranges == null ? null : ranges.copy()); + return new DeletionInfo(partitionDeletion, ranges == null ? null : ranges.copy()); } public DeletionInfo copy(AbstractAllocator allocator) { - RangeTombstoneList rangesCopy = null; if (ranges != null) rangesCopy = ranges.copy(allocator); - return new DeletionInfo(topLevel, rangesCopy); + return new DeletionInfo(partitionDeletion, rangesCopy); } /** @@ -118,106 +104,31 @@ public class DeletionInfo implements IMeasurableMemory */ public boolean isLive() { - return topLevel.isLive() && (ranges == null || ranges.isEmpty()); + return partitionDeletion.isLive() && (ranges == null || ranges.isEmpty()); } /** - * Return whether a given cell is deleted by the container having this deletion info. + * Return whether a given cell is deleted by this deletion info. * + * @param clustering the clustering for the cell to check. * @param cell the cell to check. * @return true if the cell is deleted, false otherwise */ - public boolean isDeleted(Cell cell) + private boolean isDeleted(Clustering clustering, Cell cell) { - // We do rely on this test: if topLevel.markedForDeleteAt is MIN_VALUE, we should not - // consider the column deleted even if timestamp=MIN_VALUE, otherwise this break QueryFilter.isRelevant + // If we're live, don't consider anything deleted, even if the cell ends up having as timestamp Long.MIN_VALUE + // (which shouldn't happen in practice, but it would invalid to consider it deleted if it does). if (isLive()) return false; - if (cell.timestamp() <= topLevel.markedForDeleteAt) + if (cell.livenessInfo().timestamp() <= partitionDeletion.markedForDeleteAt()) return true; // No matter what the counter cell's timestamp is, a tombstone always takes precedence. See CASSANDRA-7346. - if (!topLevel.isLive() && cell instanceof CounterCell) + if (!partitionDeletion.isLive() && cell.isCounterCell()) return true; - return ranges != null && ranges.isDeleted(cell); - } - - /** - * Returns a new {@link InOrderTester} in forward order. - */ - public InOrderTester inOrderTester() - { - return inOrderTester(false); - } - - /** - * Returns a new {@link InOrderTester} given the order in which - * columns will be passed to it. - */ - public InOrderTester inOrderTester(boolean reversed) - { - return new InOrderTester(reversed); - } - - /** - * Purge every tombstones that are older than {@code gcbefore}. - * - * @param gcBefore timestamp (in seconds) before which tombstones should be purged - */ - public void purge(int gcBefore) - { - topLevel = topLevel.localDeletionTime < gcBefore ? DeletionTime.LIVE : topLevel; - - if (ranges != null) - { - ranges.purge(gcBefore); - if (ranges.isEmpty()) - ranges = null; - } - } - - /** - * Evaluates difference between this deletion info and superset for read repair - * - * @return the difference between the two, or LIVE if no difference - */ - public DeletionInfo diff(DeletionInfo superset) - { - RangeTombstoneList rangeDiff = superset.ranges == null || superset.ranges.isEmpty() - ? null - : ranges == null ? superset.ranges : ranges.diff(superset.ranges); - - return topLevel.markedForDeleteAt != superset.topLevel.markedForDeleteAt || rangeDiff != null - ? new DeletionInfo(superset.topLevel, rangeDiff) - : DeletionInfo.live(); - } - - - /** - * Digests deletion info. Used to trigger read repair on mismatch. - */ - public void updateDigest(MessageDigest digest) - { - if (topLevel.markedForDeleteAt != Long.MIN_VALUE) - digest.update(ByteBufferUtil.bytes(topLevel.markedForDeleteAt)); - - if (ranges != null) - ranges.updateDigest(digest); - } - - /** - * Returns true if {@code purge} would remove the top-level tombstone or any of the range - * tombstones, false otherwise. - * @param gcBefore timestamp (in seconds) before which tombstones should be purged - */ - public boolean hasPurgeableTombstones(int gcBefore) - { - if (topLevel.localDeletionTime < gcBefore) - return true; - - return ranges != null && ranges.hasPurgeableTombstones(gcBefore); + return ranges != null && ranges.isDeleted(clustering, cell); } /** @@ -227,11 +138,11 @@ public class DeletionInfo implements IMeasurableMemory */ public void add(DeletionTime newInfo) { - if (topLevel.markedForDeleteAt < newInfo.markedForDeleteAt) - topLevel = newInfo; + if (newInfo.supersedes(partitionDeletion)) + partitionDeletion = newInfo; } - public void add(RangeTombstone tombstone, Comparator comparator) + public void add(RangeTombstone tombstone, ClusteringComparator comparator) { if (ranges == null) ranges = new RangeTombstoneList(comparator, 1); @@ -248,7 +159,7 @@ public class DeletionInfo implements IMeasurableMemory */ public DeletionInfo add(DeletionInfo newInfo) { - add(newInfo.topLevel); + add(newInfo.partitionDeletion); if (ranges == null) ranges = newInfo.ranges == null ? null : newInfo.ranges.copy(); @@ -258,53 +169,30 @@ public class DeletionInfo implements IMeasurableMemory return this; } - /** - * Returns the minimum timestamp in any of the range tombstones or the top-level tombstone. - */ - public long minTimestamp() + public DeletionTime getPartitionDeletion() { - return ranges == null - ? topLevel.markedForDeleteAt - : Math.min(topLevel.markedForDeleteAt, ranges.minMarkedAt()); - } - - /** - * Returns the maximum timestamp in any of the range tombstones or the top-level tombstone. - */ - public long maxTimestamp() - { - return ranges == null - ? topLevel.markedForDeleteAt - : Math.max(topLevel.markedForDeleteAt, ranges.maxMarkedAt()); - } - - /** - * Returns the top-level (or "row") tombstone. - */ - public DeletionTime getTopLevelDeletion() - { - return topLevel; + return partitionDeletion; } // Use sparingly, not the most efficient thing - public Iterator rangeIterator() + public Iterator rangeIterator(boolean reversed) { - return ranges == null ? Iterators.emptyIterator() : ranges.iterator(); + return ranges == null ? Iterators.emptyIterator() : ranges.iterator(reversed); } - public Iterator rangeIterator(Composite start, Composite finish) + public Iterator rangeIterator(Slice slice, boolean reversed) { - return ranges == null ? Iterators.emptyIterator() : ranges.iterator(start, finish); + return ranges == null ? Iterators.emptyIterator() : ranges.iterator(slice, reversed); } - public RangeTombstone rangeCovering(Composite name) + public RangeTombstone rangeCovering(Clustering name) { return ranges == null ? null : ranges.search(name); } public int dataSize() { - int size = TypeSizes.NATIVE.sizeof(topLevel.markedForDeleteAt); + int size = TypeSizes.NATIVE.sizeof(partitionDeletion.markedForDeleteAt()); return size + (ranges == null ? 0 : ranges.dataSize()); } @@ -323,45 +211,43 @@ public class DeletionInfo implements IMeasurableMemory */ public boolean mayModify(DeletionInfo delInfo) { - return topLevel.compareTo(delInfo.topLevel) > 0 || hasRanges(); + return partitionDeletion.compareTo(delInfo.partitionDeletion) > 0 || hasRanges(); } @Override public String toString() { if (ranges == null || ranges.isEmpty()) - return String.format("{%s}", topLevel); + return String.format("{%s}", partitionDeletion); else - return String.format("{%s, ranges=%s}", topLevel, rangesAsString()); + return String.format("{%s, ranges=%s}", partitionDeletion, rangesAsString()); } private String rangesAsString() { assert !ranges.isEmpty(); StringBuilder sb = new StringBuilder(); - CType type = (CType)ranges.comparator(); - assert type != null; - Iterator iter = rangeIterator(); + ClusteringComparator cc = ranges.comparator(); + Iterator iter = rangeIterator(false); while (iter.hasNext()) { RangeTombstone i = iter.next(); - sb.append("["); - sb.append(type.getString(i.min)).append("-"); - sb.append(type.getString(i.max)).append(", "); - sb.append(i.data); - sb.append("]"); + sb.append(i.deletedSlice().toString(cc)); + sb.append("@"); + sb.append(i.deletionTime()); } return sb.toString(); } // Updates all the timestamp of the deletion contained in this DeletionInfo to be {@code timestamp}. - public void updateAllTimestamp(long timestamp) + public DeletionInfo updateAllTimestamp(long timestamp) { - if (topLevel.markedForDeleteAt != Long.MIN_VALUE) - topLevel = new DeletionTime(timestamp, topLevel.localDeletionTime); + if (partitionDeletion.markedForDeleteAt() != Long.MIN_VALUE) + partitionDeletion = new SimpleDeletionTime(timestamp, partitionDeletion.localDeletionTime()); if (ranges != null) ranges.updateAllTimestamp(timestamp); + return this; } @Override @@ -370,100 +256,18 @@ public class DeletionInfo implements IMeasurableMemory if(!(o instanceof DeletionInfo)) return false; DeletionInfo that = (DeletionInfo)o; - return topLevel.equals(that.topLevel) && Objects.equal(ranges, that.ranges); + return partitionDeletion.equals(that.partitionDeletion) && Objects.equal(ranges, that.ranges); } @Override public final int hashCode() { - return Objects.hashCode(topLevel, ranges); + return Objects.hashCode(partitionDeletion, ranges); } @Override public long unsharedHeapSize() { - return EMPTY_SIZE + topLevel.unsharedHeapSize() + (ranges == null ? 0 : ranges.unsharedHeapSize()); - } - - public static class Serializer implements IVersionedSerializer - { - private final RangeTombstoneList.Serializer rtlSerializer; - - public Serializer(CType type) - { - this.rtlSerializer = new RangeTombstoneList.Serializer(type); - } - - public void serialize(DeletionInfo info, DataOutputPlus out, int version) throws IOException - { - DeletionTime.serializer.serialize(info.topLevel, out); - rtlSerializer.serialize(info.ranges, out, version); - } - - public DeletionInfo deserialize(DataInput in, int version) throws IOException - { - DeletionTime topLevel = DeletionTime.serializer.deserialize(in); - RangeTombstoneList ranges = rtlSerializer.deserialize(in, version); - return new DeletionInfo(topLevel, ranges); - } - - public long serializedSize(DeletionInfo info, TypeSizes typeSizes, int version) - { - long size = DeletionTime.serializer.serializedSize(info.topLevel, typeSizes); - return size + rtlSerializer.serializedSize(info.ranges, typeSizes, version); - } - - public long serializedSize(DeletionInfo info, int version) - { - return serializedSize(info, TypeSizes.NATIVE, version); - } - } - - /** - * This object allow testing whether a given column (name/timestamp) is deleted - * or not by this DeletionInfo, assuming that the columns given to this - * object are passed in forward or reversed comparator sorted order. - * - * This is more efficient that calling DeletionInfo.isDeleted() repeatedly - * in that case. - */ - public class InOrderTester - { - /* - * Note that because because range tombstone are added to this DeletionInfo while we iterate, - * `ranges` may be null initially and we need to wait for the first range to create the tester (once - * created the test will pick up new tombstones however). We are guaranteed that a range tombstone - * will be added *before* we test any column that it may delete, so this is ok. - */ - private RangeTombstoneList.InOrderTester tester; - private final boolean reversed; - - private InOrderTester(boolean reversed) - { - this.reversed = reversed; - } - - public boolean isDeleted(Cell cell) - { - if (cell.timestamp() <= topLevel.markedForDeleteAt) - return true; - - // No matter what the counter cell's timestamp is, a tombstone always takes precedence. See CASSANDRA-7346. - if (!topLevel.isLive() && cell instanceof CounterCell) - return true; - - /* - * We don't optimize the reversed case for now because RangeTombstoneList - * is always in forward sorted order. - */ - if (reversed) - return DeletionInfo.this.isDeleted(cell); - - // Maybe create the tester if we hadn't yet and we now have some ranges (see above). - if (tester == null && ranges != null) - tester = ranges.inOrderTester(); - - return tester != null && tester.isDeleted(cell); - } + return EMPTY_SIZE + partitionDeletion.unsharedHeapSize() + (ranges == null ? 0 : ranges.unsharedHeapSize()); } } diff --git a/src/java/org/apache/cassandra/db/DeletionTime.java b/src/java/org/apache/cassandra/db/DeletionTime.java index 71654172a2..f070778716 100644 --- a/src/java/org/apache/cassandra/db/DeletionTime.java +++ b/src/java/org/apache/cassandra/db/DeletionTime.java @@ -19,58 +19,56 @@ package org.apache.cassandra.db; import java.io.DataInput; import java.io.IOException; +import java.security.MessageDigest; -import com.google.common.annotations.VisibleForTesting; import com.google.common.base.Objects; import org.apache.cassandra.cache.IMeasurableMemory; import org.apache.cassandra.io.ISerializer; import org.apache.cassandra.io.util.DataOutputPlus; import org.apache.cassandra.io.util.FileUtils; +import org.apache.cassandra.utils.FBUtilities; import org.apache.cassandra.utils.ObjectSizes; -import org.codehaus.jackson.annotate.JsonIgnore; /** - * A top-level (row) tombstone. + * Information on deletion of a storage engine object. */ -public class DeletionTime implements Comparable, IMeasurableMemory +public abstract class DeletionTime implements Comparable, IMeasurableMemory, Aliasable { - private static final long EMPTY_SIZE = ObjectSizes.measure(new DeletionTime(0, 0)); + private static final long EMPTY_SIZE = ObjectSizes.measure(new SimpleDeletionTime(0, 0)); /** * A special DeletionTime that signifies that there is no top-level (row) tombstone. */ - public static final DeletionTime LIVE = new DeletionTime(Long.MIN_VALUE, Integer.MAX_VALUE); + public static final DeletionTime LIVE = new SimpleDeletionTime(Long.MIN_VALUE, Integer.MAX_VALUE); + + public static final Serializer serializer = new Serializer(); /** * A timestamp (typically in microseconds since the unix epoch, although this is not enforced) after which * data should be considered deleted. If set to Long.MIN_VALUE, this implies that the data has not been marked * for deletion at all. */ - public final long markedForDeleteAt; + public abstract long markedForDeleteAt(); /** * The local server timestamp, in seconds since the unix epoch, at which this tombstone was created. This is * only used for purposes of purging the tombstone after gc_grace_seconds have elapsed. */ - public final int localDeletionTime; - - public static final Serializer serializer = new Serializer(); - - @VisibleForTesting - public DeletionTime(long markedForDeleteAt, int localDeletionTime) - { - this.markedForDeleteAt = markedForDeleteAt; - this.localDeletionTime = localDeletionTime; - } + public abstract int localDeletionTime(); /** * Returns whether this DeletionTime is live, that is deletes no columns. */ - @JsonIgnore public boolean isLive() { - return markedForDeleteAt == Long.MIN_VALUE && localDeletionTime == Integer.MAX_VALUE; + return markedForDeleteAt() == Long.MIN_VALUE && localDeletionTime() == Integer.MAX_VALUE; + } + + public void digest(MessageDigest digest) + { + FBUtilities.updateWithLong(digest, markedForDeleteAt()); + FBUtilities.updateWithInt(digest, localDeletionTime()); } @Override @@ -79,48 +77,58 @@ public class DeletionTime implements Comparable, IMeasurableMemory if(!(o instanceof DeletionTime)) return false; DeletionTime that = (DeletionTime)o; - return markedForDeleteAt == that.markedForDeleteAt && localDeletionTime == that.localDeletionTime; + return markedForDeleteAt() == that.markedForDeleteAt() && localDeletionTime() == that.localDeletionTime(); } @Override public final int hashCode() { - return Objects.hashCode(markedForDeleteAt, localDeletionTime); + return Objects.hashCode(markedForDeleteAt(), localDeletionTime()); } @Override public String toString() { - return String.format("deletedAt=%d, localDeletion=%d", markedForDeleteAt, localDeletionTime); + return String.format("deletedAt=%d, localDeletion=%d", markedForDeleteAt(), localDeletionTime()); } public int compareTo(DeletionTime dt) { - if (markedForDeleteAt < dt.markedForDeleteAt) + if (markedForDeleteAt() < dt.markedForDeleteAt()) return -1; - else if (markedForDeleteAt > dt.markedForDeleteAt) + else if (markedForDeleteAt() > dt.markedForDeleteAt()) return 1; - else if (localDeletionTime < dt.localDeletionTime) + else if (localDeletionTime() < dt.localDeletionTime()) return -1; - else if (localDeletionTime > dt.localDeletionTime) + else if (localDeletionTime() > dt.localDeletionTime()) return -1; else return 0; } - public boolean isGcAble(int gcBefore) - { - return localDeletionTime < gcBefore; - } - - public boolean isDeleted(OnDiskAtom atom) - { - return atom.timestamp() <= markedForDeleteAt; - } - public boolean supersedes(DeletionTime dt) { - return this.markedForDeleteAt > dt.markedForDeleteAt; + return markedForDeleteAt() > dt.markedForDeleteAt() || (markedForDeleteAt() == dt.markedForDeleteAt() && localDeletionTime() > dt.localDeletionTime()); + } + + public boolean isPurgeable(long maxPurgeableTimestamp, int gcBefore) + { + return markedForDeleteAt() < maxPurgeableTimestamp && localDeletionTime() < gcBefore; + } + + public boolean deletes(LivenessInfo info) + { + return deletes(info.timestamp()); + } + + public boolean deletes(long timestamp) + { + return timestamp <= markedForDeleteAt(); + } + + public int dataSize() + { + return 12; } public long unsharedHeapSize() @@ -132,8 +140,8 @@ public class DeletionTime implements Comparable, IMeasurableMemory { public void serialize(DeletionTime delTime, DataOutputPlus out) throws IOException { - out.writeInt(delTime.localDeletionTime); - out.writeLong(delTime.markedForDeleteAt); + out.writeInt(delTime.localDeletionTime()); + out.writeLong(delTime.markedForDeleteAt()); } public DeletionTime deserialize(DataInput in) throws IOException @@ -142,7 +150,7 @@ public class DeletionTime implements Comparable, IMeasurableMemory long mfda = in.readLong(); return mfda == Long.MIN_VALUE && ldt == Integer.MAX_VALUE ? LIVE - : new DeletionTime(mfda, ldt); + : new SimpleDeletionTime(mfda, ldt); } public void skip(DataInput in) throws IOException @@ -152,8 +160,8 @@ public class DeletionTime implements Comparable, IMeasurableMemory public long serializedSize(DeletionTime delTime, TypeSizes typeSizes) { - return typeSizes.sizeof(delTime.localDeletionTime) - + typeSizes.sizeof(delTime.markedForDeleteAt); + return typeSizes.sizeof(delTime.localDeletionTime()) + + typeSizes.sizeof(delTime.markedForDeleteAt()); } } } diff --git a/src/java/org/apache/cassandra/db/DeletionTimeArray.java b/src/java/org/apache/cassandra/db/DeletionTimeArray.java new file mode 100644 index 0000000000..77eb9533d4 --- /dev/null +++ b/src/java/org/apache/cassandra/db/DeletionTimeArray.java @@ -0,0 +1,153 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.cassandra.db; + +import java.util.Arrays; + +import org.apache.cassandra.utils.ObjectSizes; + +/** + * Utility class to store an array of deletion times a bit efficiently. + */ +public class DeletionTimeArray +{ + private long[] markedForDeleteAts; + private int[] delTimes; + + public DeletionTimeArray(int initialCapacity) + { + this.markedForDeleteAts = new long[initialCapacity]; + this.delTimes = new int[initialCapacity]; + clear(); + } + + public void clear(int i) + { + markedForDeleteAts[i] = Long.MIN_VALUE; + delTimes[i] = Integer.MAX_VALUE; + } + + public void set(int i, DeletionTime dt) + { + this.markedForDeleteAts[i] = dt.markedForDeleteAt(); + this.delTimes[i] = dt.localDeletionTime(); + } + + public int size() + { + return markedForDeleteAts.length; + } + + public void resize(int newSize) + { + int prevSize = size(); + + markedForDeleteAts = Arrays.copyOf(markedForDeleteAts, newSize); + delTimes = Arrays.copyOf(delTimes, newSize); + + Arrays.fill(markedForDeleteAts, prevSize, newSize, Long.MIN_VALUE); + Arrays.fill(delTimes, prevSize, newSize, Integer.MAX_VALUE); + } + + public boolean supersedes(int i, DeletionTime dt) + { + return markedForDeleteAts[i] > dt.markedForDeleteAt(); + } + + public boolean supersedes(int i, int j) + { + return markedForDeleteAts[i] > markedForDeleteAts[j]; + } + + public void swap(int i, int j) + { + long m = markedForDeleteAts[j]; + int l = delTimes[j]; + + move(i, j); + + markedForDeleteAts[i] = m; + delTimes[i] = l; + } + + public void move(int i, int j) + { + markedForDeleteAts[j] = markedForDeleteAts[i]; + delTimes[j] = delTimes[i]; + } + + public boolean isLive(int i) + { + return markedForDeleteAts[i] > Long.MIN_VALUE; + } + + public void clear() + { + Arrays.fill(markedForDeleteAts, Long.MIN_VALUE); + Arrays.fill(delTimes, Integer.MAX_VALUE); + } + + public int dataSize() + { + return 12 * markedForDeleteAts.length; + } + + public long unsharedHeapSize() + { + return ObjectSizes.sizeOfArray(markedForDeleteAts) + + ObjectSizes.sizeOfArray(delTimes); + } + + public void copy(DeletionTimeArray other) + { + assert size() == other.size(); + for (int i = 0; i < size(); i++) + { + markedForDeleteAts[i] = other.markedForDeleteAts[i]; + delTimes[i] = other.delTimes[i]; + } + } + + public static class Cursor extends DeletionTime + { + private DeletionTimeArray array; + private int i; + + public Cursor setTo(DeletionTimeArray array, int i) + { + this.array = array; + this.i = i; + return this; + } + + public long markedForDeleteAt() + { + return array.markedForDeleteAts[i]; + } + + public int localDeletionTime() + { + return array.delTimes[i]; + } + + public DeletionTime takeAlias() + { + return new SimpleDeletionTime(markedForDeleteAt(), localDeletionTime()); + } + } +} diff --git a/src/java/org/apache/cassandra/db/ExpiringCell.java b/src/java/org/apache/cassandra/db/ExpiringCell.java deleted file mode 100644 index 5fc0f94592..0000000000 --- a/src/java/org/apache/cassandra/db/ExpiringCell.java +++ /dev/null @@ -1,44 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.apache.cassandra.db; - -import org.apache.cassandra.config.CFMetaData; -import org.apache.cassandra.utils.concurrent.OpOrder; -import org.apache.cassandra.utils.memory.AbstractAllocator; -import org.apache.cassandra.utils.memory.MemtableAllocator; - -/** - * Alternative to Cell that have an expiring time. - * ExpiringCell is immutable (as Cell is). - * - * Note that ExpiringCell does not override Cell.getMarkedForDeleteAt, - * which means that it's in the somewhat unintuitive position of being deleted (after its expiration) - * without having a time-at-which-it-became-deleted. (Because ttl is a server-side measurement, - * we can't mix it with the timestamp field, which is client-supplied and whose resolution we - * can't assume anything about.) - */ -public interface ExpiringCell extends Cell -{ - public static final int MAX_TTL = 20 * 365 * 24 * 60 * 60; // 20 years in seconds - - public int getTimeToLive(); - - ExpiringCell localCopy(CFMetaData metadata, AbstractAllocator allocator); - - ExpiringCell localCopy(CFMetaData metaData, MemtableAllocator allocator, OpOrder.Group opGroup); -} diff --git a/src/java/org/apache/cassandra/db/HintedHandOffManager.java b/src/java/org/apache/cassandra/db/HintedHandOffManager.java index df8820bff1..848ba01624 100644 --- a/src/java/org/apache/cassandra/db/HintedHandOffManager.java +++ b/src/java/org/apache/cassandra/db/HintedHandOffManager.java @@ -30,7 +30,6 @@ import javax.management.MBeanServer; import javax.management.ObjectName; import com.google.common.annotations.VisibleForTesting; -import com.google.common.collect.ImmutableSortedSet; import com.google.common.collect.Lists; import com.google.common.util.concurrent.RateLimiter; import com.google.common.util.concurrent.Uninterruptibles; @@ -39,17 +38,15 @@ import org.slf4j.LoggerFactory; import org.apache.cassandra.concurrent.JMXEnabledScheduledThreadPoolExecutor; import org.apache.cassandra.concurrent.NamedThreadFactory; +import org.apache.cassandra.config.ColumnDefinition; import org.apache.cassandra.config.DatabaseDescriptor; -import org.apache.cassandra.config.Schema; import org.apache.cassandra.db.compaction.CompactionManager; -import org.apache.cassandra.db.composites.*; +import org.apache.cassandra.db.rows.*; +import org.apache.cassandra.db.partitions.*; import org.apache.cassandra.db.filter.*; import org.apache.cassandra.db.marshal.Int32Type; import org.apache.cassandra.db.marshal.UUIDType; -import org.apache.cassandra.dht.IPartitioner; -import org.apache.cassandra.dht.Range; import org.apache.cassandra.dht.Token; -import org.apache.cassandra.exceptions.WriteFailureException; import org.apache.cassandra.exceptions.WriteTimeoutException; import org.apache.cassandra.gms.ApplicationState; import org.apache.cassandra.gms.FailureDetector; @@ -61,6 +58,7 @@ import org.apache.cassandra.net.MessageOut; import org.apache.cassandra.net.MessagingService; import org.apache.cassandra.service.*; import org.apache.cassandra.utils.*; +import org.apache.cassandra.utils.concurrent.OpOrder; import org.cliffc.high_scale_lib.NonBlockingHashSet; /** @@ -91,7 +89,8 @@ public class HintedHandOffManager implements HintedHandOffManagerMBean public static final HintedHandOffManager instance = new HintedHandOffManager(); private static final Logger logger = LoggerFactory.getLogger(HintedHandOffManager.class); - private static final int PAGE_SIZE = 128; + + private static final int MAX_SIMULTANEOUSLY_REPLAYED_HINTS = 128; private static final int LARGE_NUMBER = 65536; // 64k nodes ought to be enough for anybody. public final HintedHandoffMetrics metrics = new HintedHandoffMetrics(); @@ -110,6 +109,8 @@ public class HintedHandOffManager implements HintedHandOffManagerMBean private final ColumnFamilyStore hintStore = Keyspace.open(SystemKeyspace.NAME).getColumnFamilyStore(SystemKeyspace.HINTS); + private static final ColumnDefinition hintColumn = SystemKeyspace.Hints.compactValueColumn(); + /** * Returns a mutation representing a Hint to be sent to targetId * as soon as it becomes available again. @@ -127,11 +128,20 @@ public class HintedHandOffManager implements HintedHandOffManagerMBean UUID hintId = UUIDGen.getTimeUUID(); // serialize the hint with id and version as a composite column name - CellName name = SystemKeyspace.Hints.comparator.makeCellName(hintId, MessagingService.current_version); + + PartitionUpdate upd = new PartitionUpdate(SystemKeyspace.Hints, + StorageService.getPartitioner().decorateKey(UUIDType.instance.decompose(targetId)), + PartitionColumns.of(hintColumn), + 1); + + Row.Writer writer = upd.writer(); + Rows.writeClustering(SystemKeyspace.Hints.comparator.make(hintId, MessagingService.current_version), writer); + ByteBuffer value = ByteBuffer.wrap(FBUtilities.serialize(mutation, Mutation.serializer, MessagingService.current_version)); - ColumnFamily cf = ArrayBackedSortedColumns.factory.create(Schema.instance.getCFMetaData(SystemKeyspace.NAME, SystemKeyspace.HINTS)); - cf.addColumn(name, value, now, ttl); - return new Mutation(SystemKeyspace.NAME, UUIDType.instance.decompose(targetId), cf); + writer.writeCell(hintColumn, false, value, SimpleLivenessInfo.forUpdate(now, ttl, FBUtilities.nowInSeconds(), SystemKeyspace.Hints), null); + writer.endOfRow(); + + return new Mutation(upd); } /* @@ -142,12 +152,11 @@ public class HintedHandOffManager implements HintedHandOffManagerMBean public static int calculateHintTTL(Mutation mutation) { int ttl = maxHintTTL; - for (ColumnFamily cf : mutation.getColumnFamilies()) - ttl = Math.min(ttl, cf.metadata().getGcGraceSeconds()); + for (PartitionUpdate upd : mutation.getPartitionUpdates()) + ttl = Math.min(ttl, upd.metadata().getGcGraceSeconds()); return ttl; } - public void start() { MBeanServer mbs = ManagementFactory.getPlatformMBeanServer(); @@ -172,11 +181,17 @@ public class HintedHandOffManager implements HintedHandOffManagerMBean executor.scheduleWithFixedDelay(runnable, 10, 10, TimeUnit.MINUTES); } - private static void deleteHint(ByteBuffer tokenBytes, CellName columnName, long timestamp) + private static void deleteHint(ByteBuffer tokenBytes, Clustering clustering, long timestamp) { - Mutation mutation = new Mutation(SystemKeyspace.NAME, tokenBytes); - mutation.delete(SystemKeyspace.HINTS, columnName, timestamp); - mutation.applyUnsafe(); // don't bother with commitlog since we're going to flush as soon as we're done with delivery + DecoratedKey dk = StorageService.getPartitioner().decorateKey(tokenBytes); + + PartitionUpdate upd = new PartitionUpdate(SystemKeyspace.Hints, dk, PartitionColumns.of(hintColumn), 1); + + Row.Writer writer = upd.writer(); + Rows.writeClustering(clustering, writer); + Cells.writeTombstone(writer, hintColumn, timestamp, FBUtilities.nowInSeconds()); + + new Mutation(upd).applyUnsafe(); // don't bother with commitlog since we're going to flush as soon as we're done with delivery } public void deleteHintsForEndpoint(final String ipOrHostname) @@ -198,9 +213,8 @@ public class HintedHandOffManager implements HintedHandOffManagerMBean if (!StorageService.instance.getTokenMetadata().isMember(endpoint)) return; UUID hostId = StorageService.instance.getTokenMetadata().getHostId(endpoint); - ByteBuffer hostIdBytes = ByteBuffer.wrap(UUIDGen.decompose(hostId)); - final Mutation mutation = new Mutation(SystemKeyspace.NAME, hostIdBytes); - mutation.delete(SystemKeyspace.HINTS, System.currentTimeMillis()); + DecoratedKey dk = StorageService.getPartitioner().decorateKey(ByteBuffer.wrap(UUIDGen.decompose(hostId))); + final Mutation mutation = new Mutation(PartitionUpdate.fullPartitionDelete(SystemKeyspace.Hints, dk, System.currentTimeMillis(), FBUtilities.nowInSeconds())); // execute asynchronously to avoid blocking caller (which may be processing gossip) Runnable runnable = new Runnable() @@ -266,13 +280,6 @@ public class HintedHandOffManager implements HintedHandOffManagerMBean } } - private static boolean pagingFinished(ColumnFamily hintColumnFamily, Composite startColumn) - { - // done if no hints found or the start column (same as last column processed in previous iteration) is the only one - return hintColumnFamily == null - || (!startColumn.isEmpty() && hintColumnFamily.getSortedColumns().size() == 1 && hintColumnFamily.getColumn((CellName)startColumn) != null); - } - private int waitForSchemaAgreement(InetAddress endpoint) throws TimeoutException { Gossiper gossiper = Gossiper.instance; @@ -335,6 +342,27 @@ public class HintedHandOffManager implements HintedHandOffManagerMBean } doDeliverHintsToEndpoint(endpoint); + + // Flush all the tombstones to disk + hintStore.forceBlockingFlush(); + } + + private boolean checkDelivered(InetAddress endpoint, List> handlers, AtomicInteger rowsReplayed) + { + for (WriteResponseHandler handler : handlers) + { + try + { + handler.get(); + } + catch (WriteTimeoutException e) + { + logger.info("Failed replaying hints to {}; aborting ({} delivered), error : {}", + endpoint, rowsReplayed, e.getMessage()); + return false; + } + } + return true; } /* @@ -352,10 +380,6 @@ public class HintedHandOffManager implements HintedHandOffManagerMBean DecoratedKey epkey = StorageService.getPartitioner().decorateKey(hostIdBytes); final AtomicInteger rowsReplayed = new AtomicInteger(0); - Composite startColumn = Composites.EMPTY; - - int pageSize = calculatePageSize(); - logger.debug("Using pageSize of {}", pageSize); // rate limit is in bytes per second. Uses Double.MAX_VALUE if disabled (set to 0 in cassandra.yaml). // max rate is scaled by the number of nodes in the cluster (CASSANDRA-5272). @@ -363,55 +387,38 @@ public class HintedHandOffManager implements HintedHandOffManagerMBean / (StorageService.instance.getTokenMetadata().getAllEndpoints().size() - 1); RateLimiter rateLimiter = RateLimiter.create(throttleInKB == 0 ? Double.MAX_VALUE : throttleInKB * 1024); - delivery: - while (true) + int nowInSec = FBUtilities.nowInSeconds(); + try (OpOrder.Group op = hintStore.readOrdering.start(); + RowIterator iter = UnfilteredRowIterators.filter(SinglePartitionReadCommand.fullPartitionRead(SystemKeyspace.Hints, nowInSec, epkey).queryMemtableAndDisk(hintStore, op), nowInSec)) { - long now = System.currentTimeMillis(); - QueryFilter filter = QueryFilter.getSliceFilter(epkey, - SystemKeyspace.HINTS, - startColumn, - Composites.EMPTY, - false, - pageSize, - now); - - ColumnFamily hintsPage = ColumnFamilyStore.removeDeleted(hintStore.getColumnFamily(filter), (int) (now / 1000)); - - if (pagingFinished(hintsPage, startColumn)) - { - logger.info("Finished hinted handoff of {} rows to endpoint {}", rowsReplayed, endpoint); - break; - } - - // check if node is still alive and we should continue delivery process - if (!FailureDetector.instance.isAlive(endpoint)) - { - logger.info("Endpoint {} died during hint delivery; aborting ({} delivered)", endpoint, rowsReplayed); - break; - } - List> responseHandlers = Lists.newArrayList(); - for (final Cell hint : hintsPage) + + while (iter.hasNext()) { + // check if node is still alive and we should continue delivery process + if (!FailureDetector.instance.isAlive(endpoint)) + { + logger.info("Endpoint {} died during hint delivery; aborting ({} delivered)", endpoint, rowsReplayed); + return; + } + // check if hints delivery has been paused during the process if (hintedHandOffPaused) { logger.debug("Hints delivery process is paused, aborting"); - break delivery; + return; } - // Skip tombstones: - // if we iterate quickly enough, it's possible that we could request a new page in the same millisecond - // in which the local deletion timestamp was generated on the last column in the old page, in which - // case the hint will have no columns (since it's deleted) but will still be included in the resultset - // since (even with gcgs=0) it's still a "relevant" tombstone. - if (!hint.isLive()) - continue; + // Wait regularly on the endpoint acknowledgment. If we timeout on it, the endpoint is probably dead so stop delivery + if (responseHandlers.size() > MAX_SIMULTANEOUSLY_REPLAYED_HINTS && !checkDelivered(endpoint, responseHandlers, rowsReplayed)) + return; - startColumn = hint.name(); + final Row hint = iter.next(); + int version = Int32Type.instance.compose(hint.clustering().get(1)); + Cell cell = hint.getCell(hintColumn); - int version = Int32Type.instance.compose(hint.name().get(1)); - DataInputStream in = new DataInputStream(ByteBufferUtil.inputStream(hint.value())); + final long timestamp = cell.livenessInfo().timestamp(); + DataInputStream in = new DataInputStream(ByteBufferUtil.inputStream(cell.value())); Mutation mutation; try { @@ -420,7 +427,7 @@ public class HintedHandOffManager implements HintedHandOffManagerMBean catch (UnknownColumnFamilyException e) { logger.debug("Skipping delivery of hint for deleted table", e); - deleteHint(hostIdBytes, hint.name(), hint.timestamp()); + deleteHint(hostIdBytes, hint.clustering(), timestamp); continue; } catch (IOException e) @@ -430,7 +437,7 @@ public class HintedHandOffManager implements HintedHandOffManagerMBean for (UUID cfId : mutation.getColumnFamilyIds()) { - if (hint.timestamp() <= SystemKeyspace.getTruncatedAt(cfId)) + if (timestamp <= SystemKeyspace.getTruncatedAt(cfId)) { logger.debug("Skipping delivery of hint for truncated table {}", cfId); mutation = mutation.without(cfId); @@ -439,7 +446,7 @@ public class HintedHandOffManager implements HintedHandOffManagerMBean if (mutation.isEmpty()) { - deleteHint(hostIdBytes, hint.name(), hint.timestamp()); + deleteHint(hostIdBytes, hint.clustering(), timestamp); continue; } @@ -450,7 +457,7 @@ public class HintedHandOffManager implements HintedHandOffManagerMBean public void run() { rowsReplayed.incrementAndGet(); - deleteHint(hostIdBytes, hint.name(), hint.timestamp()); + deleteHint(hostIdBytes, hint.clustering(), timestamp); } }; WriteResponseHandler responseHandler = new WriteResponseHandler<>(endpoint, WriteType.SIMPLE, callback); @@ -458,38 +465,10 @@ public class HintedHandOffManager implements HintedHandOffManagerMBean responseHandlers.add(responseHandler); } - for (WriteResponseHandler handler : responseHandlers) - { - try - { - handler.get(); - } - catch (WriteTimeoutException|WriteFailureException e) - { - logger.info("Failed replaying hints to {}; aborting ({} delivered), error : {}", - endpoint, rowsReplayed, e.getMessage()); - break delivery; - } - } + // Wait on the last handlers + if (checkDelivered(endpoint, responseHandlers, rowsReplayed)) + logger.info("Finished hinted handoff of {} rows to endpoint {}", rowsReplayed, endpoint); } - - // Flush all the tombstones to disk - hintStore.forceBlockingFlush(); - } - - // read less columns (mutations) per page if they are very large - private int calculatePageSize() - { - int meanColumnCount = hintStore.getMeanColumns(); - if (meanColumnCount <= 0) - return PAGE_SIZE; - - int averageColumnSize = (int) (hintStore.metric.meanRowSize.getValue() / meanColumnCount); - if (averageColumnSize <= 0) - return PAGE_SIZE; - - // page size of 1 does not allow actual paging b/c of >= behavior on startColumn - return Math.max(2, Math.min(PAGE_SIZE, 4 * 1024 * 1024 / averageColumnSize)); } /** @@ -505,18 +484,26 @@ public class HintedHandOffManager implements HintedHandOffManagerMBean // to deliver to). compact(); - IPartitioner p = StorageService.getPartitioner(); - RowPosition minPos = p.getMinimumToken().minKeyBound(); - Range range = new Range<>(minPos, minPos); - IDiskAtomFilter filter = new NamesQueryFilter(ImmutableSortedSet.of()); - List rows = hintStore.getRangeSlice(range, null, filter, Integer.MAX_VALUE, System.currentTimeMillis()); - for (Row row : rows) + ReadCommand cmd = new PartitionRangeReadCommand(hintStore.metadata, + FBUtilities.nowInSeconds(), + ColumnFilter.all(hintStore.metadata), + RowFilter.NONE, + DataLimits.cqlLimits(Integer.MAX_VALUE, 1), + DataRange.allData(StorageService.getPartitioner())); + + try (ReadOrderGroup orderGroup = cmd.startOrderGroup(); UnfilteredPartitionIterator iter = cmd.executeLocally(orderGroup)) { - UUID hostId = UUIDGen.getUUID(row.key.getKey()); - InetAddress target = StorageService.instance.getTokenMetadata().getEndpointForHostId(hostId); - // token may have since been removed (in which case we have just read back a tombstone) - if (target != null) - scheduleHintDelivery(target, false); + while (iter.hasNext()) + { + try (UnfilteredRowIterator partition = iter.next()) + { + UUID hostId = UUIDGen.getUUID(partition.partitionKey().getKey()); + InetAddress target = StorageService.instance.getTokenMetadata().getEndpointForHostId(hostId); + // token may have since been removed (in which case we have just read back a tombstone) + if (target != null) + scheduleHintDelivery(target, false); + } + } } logger.debug("Finished scheduleAllDeliveries"); @@ -572,42 +559,20 @@ public class HintedHandOffManager implements HintedHandOffManagerMBean // Extract the keys as strings to be reported. LinkedList result = new LinkedList<>(); - for (Row row : getHintsSlice(1)) + ReadCommand cmd = PartitionRangeReadCommand.allDataRead(SystemKeyspace.Hints, FBUtilities.nowInSeconds()); + try (ReadOrderGroup orderGroup = cmd.startOrderGroup(); UnfilteredPartitionIterator iter = cmd.executeLocally(orderGroup)) { - if (row.cf != null) //ignore removed rows - result.addFirst(tokenFactory.toString(row.key.getToken())); + while (iter.hasNext()) + { + try (UnfilteredRowIterator partition = iter.next()) + { + // We don't delete by range on the hints table, so we don't have to worry about the + // iterator returning only range tombstone marker + if (partition.hasNext()) + result.addFirst(tokenFactory.toString(partition.partitionKey().getToken())); + } + } } return result; } - - private List getHintsSlice(int columnCount) - { - // Get count # of columns... - SliceQueryFilter predicate = new SliceQueryFilter(ColumnSlice.ALL_COLUMNS_ARRAY, - false, - columnCount); - - // From keys "" to ""... - IPartitioner partitioner = StorageService.getPartitioner(); - RowPosition minPos = partitioner.getMinimumToken().minKeyBound(); - Range range = new Range<>(minPos, minPos); - - try - { - RangeSliceCommand cmd = new RangeSliceCommand(SystemKeyspace.NAME, - SystemKeyspace.HINTS, - System.currentTimeMillis(), - predicate, - range, - null, - LARGE_NUMBER); - return StorageProxy.getRangeSlice(cmd, ConsistencyLevel.ONE); - } - catch (Exception e) - { - logger.info("HintsCF getEPPendingHints timed out."); - throw new RuntimeException(e); - } - } - } diff --git a/src/java/org/apache/cassandra/db/IMutation.java b/src/java/org/apache/cassandra/db/IMutation.java index 44df104c20..aad35c3bbf 100644 --- a/src/java/org/apache/cassandra/db/IMutation.java +++ b/src/java/org/apache/cassandra/db/IMutation.java @@ -21,13 +21,14 @@ import java.nio.ByteBuffer; import java.util.Collection; import java.util.UUID; +import org.apache.cassandra.db.partitions.PartitionUpdate; + public interface IMutation { public String getKeyspaceName(); public Collection getColumnFamilyIds(); - public ByteBuffer key(); + public DecoratedKey key(); public long getTimeout(); public String toString(boolean shallow); - public void addAll(IMutation m); - public Collection getColumnFamilies(); + public Collection getPartitionUpdates(); } diff --git a/src/java/org/apache/cassandra/db/IndexExpression.java b/src/java/org/apache/cassandra/db/IndexExpression.java deleted file mode 100644 index bdb74ce9d5..0000000000 --- a/src/java/org/apache/cassandra/db/IndexExpression.java +++ /dev/null @@ -1,121 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - */ - -package org.apache.cassandra.db; - -import java.io.DataInput; -import java.io.IOException; -import java.nio.ByteBuffer; - -import com.google.common.base.Objects; - -import org.apache.cassandra.cql3.Operator; -import org.apache.cassandra.io.util.DataOutputPlus; -import org.apache.cassandra.utils.ByteBufferUtil; - -public final class IndexExpression -{ - public final ByteBuffer column; - public final Operator operator; - public final ByteBuffer value; - - public IndexExpression(ByteBuffer column, Operator operator, ByteBuffer value) - { - this.column = column; - this.operator = operator; - this.value = value; - } - - /** - * Checks if the operator of this IndexExpression is a CONTAINS operator. - * - * @return true if the operator of this IndexExpression is a CONTAINS - * operator, false otherwise. - */ - public boolean isContains() - { - return Operator.CONTAINS == operator; - } - - /** - * Checks if the operator of this IndexExpression is a CONTAINS_KEY operator. - * - * @return true if the operator of this IndexExpression is a CONTAINS_KEY - * operator, false otherwise. - */ - public boolean isContainsKey() - { - return Operator.CONTAINS_KEY == operator; - } - - @Override - public String toString() - { - return String.format("%s %s %s", ByteBufferUtil.bytesToHex(column), operator, ByteBufferUtil.bytesToHex(value)); - } - - @Override - public boolean equals(Object o) - { - if (this == o) - return true; - - if (!(o instanceof IndexExpression)) - return false; - - IndexExpression ie = (IndexExpression) o; - - return Objects.equal(this.column, ie.column) - && Objects.equal(this.operator, ie.operator) - && Objects.equal(this.value, ie.value); - } - - @Override - public int hashCode() - { - return Objects.hashCode(column, operator, value); - } - - /** - * Write the serialized version of this IndexExpression to the specified output. - * - * @param output the output to write to - * @throws IOException if an I/O problem occurs while writing to the specified output - */ - public void writeTo(DataOutputPlus output) throws IOException - { - ByteBufferUtil.writeWithShortLength(column, output); - operator.writeTo(output); - ByteBufferUtil.writeWithShortLength(value, output); - } - - /** - * Deserializes an IndexExpression instance from the specified input. - * - * @param input the input to read from - * @return the IndexExpression instance deserialized - * @throws IOException if a problem occurs while deserializing the IndexExpression instance. - */ - public static IndexExpression readFrom(DataInput input) throws IOException - { - return new IndexExpression(ByteBufferUtil.readWithShortLength(input), - Operator.readFrom(input), - ByteBufferUtil.readWithShortLength(input)); - } -} diff --git a/src/java/org/apache/cassandra/db/Keyspace.java b/src/java/org/apache/cassandra/db/Keyspace.java index cb5c54ddc4..e045466b93 100644 --- a/src/java/org/apache/cassandra/db/Keyspace.java +++ b/src/java/org/apache/cassandra/db/Keyspace.java @@ -30,18 +30,19 @@ import org.slf4j.Logger; import org.slf4j.LoggerFactory; import org.apache.cassandra.config.*; +import org.apache.cassandra.db.rows.UnfilteredRowIterator; import org.apache.cassandra.db.commitlog.CommitLog; import org.apache.cassandra.db.commitlog.ReplayPosition; import org.apache.cassandra.db.compaction.CompactionManager; -import org.apache.cassandra.db.filter.QueryFilter; import org.apache.cassandra.db.index.SecondaryIndex; import org.apache.cassandra.db.index.SecondaryIndexManager; +import org.apache.cassandra.db.partitions.PartitionUpdate; import org.apache.cassandra.io.sstable.format.SSTableReader; import org.apache.cassandra.locator.AbstractReplicationStrategy; import org.apache.cassandra.service.StorageService; -import org.apache.cassandra.service.pager.QueryPagers; import org.apache.cassandra.tracing.Tracing; import org.apache.cassandra.utils.concurrent.OpOrder; +import org.apache.cassandra.utils.FBUtilities; import org.apache.cassandra.metrics.KeyspaceMetrics; /** @@ -49,8 +50,6 @@ import org.apache.cassandra.metrics.KeyspaceMetrics; */ public class Keyspace { - private static final int DEFAULT_PAGE_SIZE = 10000; - private static final Logger logger = LoggerFactory.getLogger(Keyspace.class); private static final String TEST_FAIL_WRITES_KS = System.getProperty("cassandra.test.fail_writes_ks", ""); @@ -145,6 +144,11 @@ public class Keyspace } } + public static ColumnFamilyStore openAndGetStore(CFMetaData cfm) + { + return open(cfm.ksName).getColumnFamilyStore(cfm.cfId); + } + /** * Removes every SSTable in the directory from the appropriate Tracker's view. * @param directory the unreadable directory, possibly with SSTables in it, but not necessarily. @@ -347,13 +351,6 @@ public class Keyspace } } - public Row getRow(QueryFilter filter) - { - ColumnFamilyStore cfStore = getColumnFamilyStore(filter.getColumnFamilyName()); - ColumnFamily columnFamily = cfStore.getColumnFamily(filter); - return new Row(filter.key, columnFamily); - } - public void apply(Mutation mutation, boolean writeCommitLog) { apply(mutation, writeCommitLog, true); @@ -372,6 +369,7 @@ public class Keyspace if (TEST_FAIL_WRITES && metadata.name.equals(TEST_FAIL_WRITES_KS)) throw new RuntimeException("Testing write failures"); + int nowInSec = FBUtilities.nowInSeconds(); try (OpOrder.Group opGroup = writeOrder.start()) { // write the mutation to the commitlog and memtables @@ -382,21 +380,21 @@ public class Keyspace replayPosition = CommitLog.instance.add(mutation); } - DecoratedKey key = StorageService.getPartitioner().decorateKey(mutation.key()); - for (ColumnFamily cf : mutation.getColumnFamilies()) + DecoratedKey key = mutation.key(); + for (PartitionUpdate upd : mutation.getPartitionUpdates()) { - ColumnFamilyStore cfs = columnFamilyStores.get(cf.id()); + ColumnFamilyStore cfs = columnFamilyStores.get(upd.metadata().cfId); if (cfs == null) { - logger.error("Attempting to mutate non-existant table {}", cf.id()); + logger.error("Attempting to mutate non-existant table {}", upd.metadata().cfId); continue; } - Tracing.trace("Adding to {} memtable", cf.metadata().cfName); + Tracing.trace("Adding to {} memtable", upd.metadata().cfName); SecondaryIndexManager.Updater updater = updateIndexes - ? cfs.indexManager.updaterFor(key, cf, opGroup) + ? cfs.indexManager.updaterFor(upd, opGroup, nowInSec) : SecondaryIndexManager.nullUpdater; - cfs.apply(key, cf, updater, opGroup, replayPosition); + cfs.apply(upd, updater, opGroup, replayPosition); } } } @@ -408,30 +406,21 @@ public class Keyspace /** * @param key row to index - * @param cfs ColumnFamily to index row in + * @param cfs ColumnFamily to index partition in * @param idxNames columns to index, in comparator order */ - public static void indexRow(DecoratedKey key, ColumnFamilyStore cfs, Set idxNames) + public static void indexPartition(DecoratedKey key, ColumnFamilyStore cfs, Set idxNames) { if (logger.isDebugEnabled()) - logger.debug("Indexing row {} ", cfs.metadata.getKeyValidator().getString(key.getKey())); + logger.debug("Indexing partition {} ", cfs.metadata.getKeyValidator().getString(key.getKey())); - try (OpOrder.Group opGroup = cfs.keyspace.writeOrder.start()) + Set indexes = cfs.indexManager.getIndexesByNames(idxNames); + SinglePartitionReadCommand cmd = SinglePartitionReadCommand.fullPartitionRead(cfs.metadata, FBUtilities.nowInSeconds(), key); + + try (OpOrder.Group opGroup = cfs.keyspace.writeOrder.start(); + UnfilteredRowIterator partition = cmd.queryMemtableAndDisk(cfs, opGroup)) { - Set indexes = cfs.indexManager.getIndexesByNames(idxNames); - - Iterator pager = QueryPagers.pageRowLocally(cfs, key.getKey(), DEFAULT_PAGE_SIZE); - while (pager.hasNext()) - { - ColumnFamily cf = pager.next(); - ColumnFamily cf2 = cf.cloneMeShallow(); - for (Cell cell : cf) - { - if (cfs.indexManager.indexes(cell.name(), indexes)) - cf2.addColumn(cell); - } - cfs.indexManager.indexRow(key.getKey(), cf2, opGroup); - } + cfs.indexManager.indexPartition(partition, opGroup, indexes, cmd.nowInSec()); } } diff --git a/src/java/org/apache/cassandra/db/LegacyLayout.java b/src/java/org/apache/cassandra/db/LegacyLayout.java new file mode 100644 index 0000000000..c1a7fd020e --- /dev/null +++ b/src/java/org/apache/cassandra/db/LegacyLayout.java @@ -0,0 +1,1301 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.cassandra.db; + +import java.io.DataInput; +import java.io.IOException; +import java.io.IOError; +import java.nio.ByteBuffer; +import java.util.*; + +import com.google.common.collect.AbstractIterator; +import com.google.common.collect.Iterators; +import com.google.common.collect.PeekingIterator; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import org.apache.cassandra.config.CFMetaData; +import org.apache.cassandra.config.ColumnDefinition; +import org.apache.cassandra.db.rows.*; +import org.apache.cassandra.db.partitions.*; +import org.apache.cassandra.db.context.CounterContext; +import org.apache.cassandra.db.marshal.*; +import org.apache.cassandra.io.util.DataOutputPlus; +import org.apache.cassandra.thrift.ColumnDef; +import org.apache.cassandra.utils.*; +import org.apache.hadoop.io.serializer.Serialization; + +/** + * Functions to deal with the old format. + */ +public abstract class LegacyLayout +{ + private static final Logger logger = LoggerFactory.getLogger(LegacyLayout.class); + + public final static int MAX_CELL_NAME_LENGTH = FBUtilities.MAX_UNSIGNED_SHORT; + + private final static int DELETION_MASK = 0x01; + private final static int EXPIRATION_MASK = 0x02; + private final static int COUNTER_MASK = 0x04; + private final static int COUNTER_UPDATE_MASK = 0x08; + private final static int RANGE_TOMBSTONE_MASK = 0x10; + + private LegacyLayout() {} + + public static AbstractType makeLegacyComparator(CFMetaData metadata) + { + ClusteringComparator comparator = metadata.comparator; + if (!metadata.isCompound()) + { + assert comparator.size() == 1; + return comparator.subtype(0); + } + + boolean hasCollections = metadata.hasCollectionColumns(); + List> types = new ArrayList<>(comparator.size() + (metadata.isDense() ? 0 : 1) + (hasCollections ? 1 : 0)); + + types.addAll(comparator.subtypes()); + + if (!metadata.isDense()) + { + types.add(UTF8Type.instance); + if (hasCollections) + { + Map defined = new HashMap<>(); + for (ColumnDefinition def : metadata.partitionColumns()) + { + if (def.type instanceof CollectionType && def.type.isMultiCell()) + defined.put(def.name.bytes, (CollectionType)def.type); + } + types.add(ColumnToCollectionType.getInstance(defined)); + } + } + return CompositeType.getInstance(types); + } + + public static LegacyCellName decodeCellName(CFMetaData metadata, ByteBuffer superColumnName, ByteBuffer cellname) + throws UnknownColumnException + { + assert cellname != null; + if (metadata.isSuper()) + { + assert superColumnName != null; + return decodeForSuperColumn(metadata, new SimpleClustering(superColumnName), cellname); + } + + assert superColumnName == null; + return decodeCellName(metadata, cellname); + } + + private static LegacyCellName decodeForSuperColumn(CFMetaData metadata, Clustering clustering, ByteBuffer subcol) + { + ColumnDefinition def = metadata.getColumnDefinition(subcol); + if (def != null) + { + // it's a statically defined subcolumn + return new LegacyCellName(clustering, def, null); + } + + def = metadata.compactValueColumn(); + assert def != null && def.type instanceof MapType; + return new LegacyCellName(clustering, def, subcol); + } + + public static LegacyCellName decodeCellName(CFMetaData metadata, ByteBuffer cellname) throws UnknownColumnException + { + return decodeCellName(metadata, cellname, false); + } + + public static LegacyCellName decodeCellName(CFMetaData metadata, ByteBuffer cellname, boolean readAllAsDynamic) throws UnknownColumnException + { + Clustering clustering = decodeClustering(metadata, cellname); + + if (metadata.isSuper()) + return decodeForSuperColumn(metadata, clustering, CompositeType.extractComponent(cellname, 1)); + + if (metadata.isDense() || (metadata.isCompactTable() && readAllAsDynamic)) + return new LegacyCellName(clustering, metadata.compactValueColumn(), null); + + ByteBuffer column = metadata.isCompound() ? CompositeType.extractComponent(cellname, metadata.comparator.size()) : cellname; + if (column == null) + throw new IllegalArgumentException("No column name component found in cell name"); + + // Row marker, this is ok + if (!column.hasRemaining()) + return new LegacyCellName(clustering, null, null); + + ColumnDefinition def = metadata.getColumnDefinition(column); + if (def == null) + { + // If it's a compact table, it means the column is in fact a "dynamic" one + if (metadata.isCompactTable()) + return new LegacyCellName(new SimpleClustering(column), metadata.compactValueColumn(), null); + + throw new UnknownColumnException(metadata, column); + } + + ByteBuffer collectionElement = metadata.isCompound() ? CompositeType.extractComponent(cellname, metadata.comparator.size() + 1) : null; + + // Note that because static compact columns are translated to static defs in the new world order, we need to force a static + // clustering if the definition is static (as it might not be in this case). + return new LegacyCellName(def.isStatic() ? Clustering.STATIC_CLUSTERING : clustering, def, collectionElement); + } + + public static LegacyBound decodeBound(CFMetaData metadata, ByteBuffer bound, boolean isStart) + { + if (!bound.hasRemaining()) + return isStart ? LegacyBound.BOTTOM : LegacyBound.TOP; + + List components = metadata.isCompound() + ? CompositeType.splitName(bound) + : Collections.singletonList(bound); + + // Either it's a prefix of the clustering, or it's the bound of a collection range tombstone (and thus has + // the collection column name) + assert components.size() <= metadata.comparator.size() || (!metadata.isCompactTable() && components.size() == metadata.comparator.size() + 1); + + List prefix = components.size() <= metadata.comparator.size() ? components : components.subList(0, metadata.comparator.size()); + Slice.Bound sb = Slice.Bound.create(isStart ? Slice.Bound.Kind.INCL_START_BOUND : Slice.Bound.Kind.INCL_END_BOUND, + prefix.toArray(new ByteBuffer[prefix.size()])); + + ColumnDefinition collectionName = components.size() == metadata.comparator.size() + 1 + ? metadata.getColumnDefinition(components.get(metadata.comparator.size())) + : null; + return new LegacyBound(sb, metadata.isCompound() && CompositeType.isStaticName(bound), collectionName); + } + + public static ByteBuffer encodeCellName(CFMetaData metadata, Clustering clustering, ByteBuffer columnName, ByteBuffer collectionElement) + { + boolean isStatic = clustering == Clustering.STATIC_CLUSTERING; + + if (!metadata.isCompound()) + { + if (isStatic) + return columnName; + + assert clustering.size() == 1; + return clustering.get(0); + } + + // We use comparator.size() rather than clustering.size() because of static clusterings + int clusteringSize = metadata.comparator.size(); + int size = clusteringSize + (metadata.isDense() ? 0 : 1) + (collectionElement == null ? 0 : 1); + ByteBuffer[] values = new ByteBuffer[size]; + for (int i = 0; i < clusteringSize; i++) + { + if (isStatic) + { + values[i] = ByteBufferUtil.EMPTY_BYTE_BUFFER; + continue; + } + + ByteBuffer v = clustering.get(i); + // we can have null (only for dense compound tables for backward compatibility reasons) but that + // means we're done and should stop there as far as building the composite is concerned. + if (v == null) + return CompositeType.build(Arrays.copyOfRange(values, 0, i)); + + values[i] = v; + } + + if (!metadata.isDense()) + values[clusteringSize] = columnName; + if (collectionElement != null) + values[clusteringSize + 1] = collectionElement; + + return CompositeType.build(isStatic, values); + } + + public static Clustering decodeClustering(CFMetaData metadata, ByteBuffer value) + { + int csize = metadata.comparator.size(); + if (csize == 0) + return Clustering.EMPTY; + + if (metadata.isCompound() && CompositeType.isStaticName(value)) + return Clustering.STATIC_CLUSTERING; + + List components = metadata.isCompound() + ? CompositeType.splitName(value) + : Collections.singletonList(value); + + return new SimpleClustering(components.subList(0, Math.min(csize, components.size())).toArray(new ByteBuffer[csize])); + } + + public static ByteBuffer encodeClustering(CFMetaData metadata, Clustering clustering) + { + if (!metadata.isCompound()) + { + assert clustering.size() == 1; + return clustering.get(0); + } + + ByteBuffer[] values = new ByteBuffer[clustering.size()]; + for (int i = 0; i < clustering.size(); i++) + values[i] = clustering.get(i); + return CompositeType.build(values); + } + + // For serializing to old wire format + public static Pair> fromUnfilteredRowIterator(UnfilteredRowIterator iterator) + { + // we need to extract the range tombstone so materialize the partition. Since this is + // used for the on-wire format, this is not worst than it used to be. + final ArrayBackedPartition partition = ArrayBackedPartition.create(iterator); + DeletionInfo info = partition.deletionInfo(); + Iterator cells = fromRowIterator(partition.metadata(), partition.iterator(), partition.staticRow()); + return Pair.create(info, cells); + } + + // For thrift sake + public static UnfilteredRowIterator toUnfilteredRowIterator(CFMetaData metadata, + DecoratedKey key, + DeletionInfo delInfo, + Iterator cells) + { + SerializationHelper helper = new SerializationHelper(0, SerializationHelper.Flag.LOCAL); + return toUnfilteredRowIterator(metadata, key, LegacyDeletionInfo.from(delInfo), cells, false, helper); + } + + // For deserializing old wire format + public static UnfilteredRowIterator onWireCellstoUnfilteredRowIterator(CFMetaData metadata, + DecoratedKey key, + LegacyDeletionInfo delInfo, + Iterator cells, + boolean reversed, + SerializationHelper helper) + { + // If the table is a static compact, the "column_metadata" are now internally encoded as + // static. This has already been recognized by decodeCellName, but it means the cells + // provided are not in the expected order (the "static" cells are not necessarily at the front). + // So sort them to make sure toUnfilteredRowIterator works as expected. + // Further, if the query is reversed, then the on-wire format still has cells in non-reversed + // order, but we need to have them reverse in the final UnfilteredRowIterator. So reverse them. + if (metadata.isStaticCompactTable() || reversed) + { + List l = new ArrayList<>(); + Iterators.addAll(l, cells); + Collections.sort(l, legacyCellComparator(metadata, reversed)); + cells = l.iterator(); + } + + return toUnfilteredRowIterator(metadata, key, delInfo, cells, reversed, helper); + } + + private static UnfilteredRowIterator toUnfilteredRowIterator(CFMetaData metadata, + DecoratedKey key, + LegacyDeletionInfo delInfo, + Iterator cells, + boolean reversed, + SerializationHelper helper) + { + // Check if we have some static + PeekingIterator iter = Iterators.peekingIterator(cells); + Row staticRow = iter.hasNext() && iter.peek().name.clustering == Clustering.STATIC_CLUSTERING + ? getNextRow(CellGrouper.staticGrouper(metadata, helper), iter) + : Rows.EMPTY_STATIC_ROW; + + Iterator rows = convertToRows(new CellGrouper(metadata, helper), iter, delInfo); + Iterator ranges = delInfo.deletionInfo.rangeIterator(reversed); + final Iterator atoms = new RowAndTombstoneMergeIterator(metadata.comparator, reversed) + .setTo(rows, ranges); + + return new AbstractUnfilteredRowIterator(metadata, + key, + delInfo.deletionInfo.getPartitionDeletion(), + metadata.partitionColumns(), + staticRow, + reversed, + RowStats.NO_STATS) + { + protected Unfiltered computeNext() + { + return atoms.hasNext() ? atoms.next() : endOfData(); + } + }; + } + + public static Row extractStaticColumns(CFMetaData metadata, DataInput in, Columns statics) throws IOException + { + assert !statics.isEmpty(); + assert metadata.isCompactTable(); + + if (metadata.isSuper()) + // TODO: there is in practice nothing to do here, but we need to handle the column_metadata for super columns somewhere else + throw new UnsupportedOperationException(); + + Set columnsToFetch = new HashSet<>(statics.columnCount()); + for (ColumnDefinition column : statics) + columnsToFetch.add(column.name.bytes); + + StaticRow.Builder builder = StaticRow.builder(statics, false, metadata.isCounter()); + + boolean foundOne = false; + LegacyAtom atom; + while ((atom = readLegacyAtom(metadata, in, false)) != null) + { + if (atom.isCell()) + { + LegacyCell cell = atom.asCell(); + if (!columnsToFetch.contains(cell.name.encode(metadata))) + continue; + + foundOne = true; + builder.writeCell(cell.name.column, cell.isCounter(), cell.value, livenessInfo(metadata, cell), null); + } + else + { + LegacyRangeTombstone tombstone = atom.asRangeTombstone(); + // TODO: we need to track tombstones and potentially ignore cells that are + // shadowed (or even better, replace them by tombstones). + throw new UnsupportedOperationException(); + } + } + + return foundOne ? builder.build() : Rows.EMPTY_STATIC_ROW; + } + + private static Row getNextRow(CellGrouper grouper, PeekingIterator cells) + { + if (!cells.hasNext()) + return null; + + grouper.reset(); + while (cells.hasNext() && grouper.addAtom(cells.peek())) + { + // We've added the cell already in the grouper, so just skip it + cells.next(); + } + return grouper.getRow(); + } + + @SuppressWarnings("unchecked") + private static Iterator asLegacyAtomIterator(Iterator iter) + { + return (Iterator)iter; + } + + private static Iterator convertToRows(final CellGrouper grouper, final Iterator cells, final LegacyDeletionInfo delInfo) + { + // A reducer that basically does nothing, we know the 2 merge iterators can't have conflicting atoms. + MergeIterator.Reducer reducer = new MergeIterator.Reducer() + { + private LegacyAtom atom; + + public void reduce(int idx, LegacyAtom current) + { + // We're merging cell with range tombstones, so we should always only have a single atom to reduce. + assert atom == null; + atom = current; + } + + protected LegacyAtom getReduced() + { + return atom; + } + + protected void onKeyChange() + { + atom = null; + } + }; + List> iterators = Arrays.asList(asLegacyAtomIterator(cells), asLegacyAtomIterator(delInfo.inRowRangeTombstones())); + Iterator merged = MergeIterator.get(iterators, grouper.metadata.comparator, reducer); + final PeekingIterator atoms = Iterators.peekingIterator(merged); + + return new AbstractIterator() + { + protected Row computeNext() + { + if (!atoms.hasNext()) + return endOfData(); + + return getNextRow(grouper, atoms); + } + }; + } + + public static Iterator fromRowIterator(final RowIterator iterator) + { + return fromRowIterator(iterator.metadata(), iterator, iterator.staticRow()); + } + + public static Iterator fromRowIterator(final CFMetaData metadata, final Iterator iterator, final Row staticRow) + { + return new AbstractIterator() + { + private Iterator currentRow = staticRow.isEmpty() + ? Collections.emptyIterator() + : fromRow(metadata, staticRow); + + protected LegacyCell computeNext() + { + if (currentRow.hasNext()) + return currentRow.next(); + + if (!iterator.hasNext()) + return endOfData(); + + currentRow = fromRow(metadata, iterator.next()); + return computeNext(); + } + }; + } + + private static Iterator fromRow(final CFMetaData metadata, final Row row) + { + return new AbstractIterator() + { + private final Iterator cells = row.iterator(); + // we don't have (and shouldn't have) row markers for compact tables. + private boolean hasReturnedRowMarker = metadata.isCompactTable(); + + protected LegacyCell computeNext() + { + if (!hasReturnedRowMarker) + { + hasReturnedRowMarker = true; + LegacyCellName cellName = new LegacyCellName(row.clustering(), null, null); + LivenessInfo info = row.primaryKeyLivenessInfo(); + return new LegacyCell(LegacyCell.Kind.REGULAR, cellName, ByteBufferUtil.EMPTY_BYTE_BUFFER, info.timestamp(), info.localDeletionTime(), info.ttl()); + } + + if (!cells.hasNext()) + return endOfData(); + + Cell cell = cells.next(); + return makeLegacyCell(row.clustering(), cell); + } + }; + } + + private static LegacyCell makeLegacyCell(Clustering clustering, Cell cell) + { + LegacyCell.Kind kind; + if (cell.isCounterCell()) + kind = LegacyCell.Kind.COUNTER; + else if (cell.isTombstone()) + kind = LegacyCell.Kind.DELETED; + else if (cell.isExpiring()) + kind = LegacyCell.Kind.EXPIRING; + else + kind = LegacyCell.Kind.REGULAR; + + CellPath path = cell.path(); + assert path == null || path.size() == 1; + LegacyCellName name = new LegacyCellName(clustering, cell.column(), path == null ? null : path.get(0)); + LivenessInfo info = cell.livenessInfo(); + return new LegacyCell(kind, name, cell.value(), info.timestamp(), info.localDeletionTime(), info.ttl()); + } + + public static RowIterator toRowIterator(final CFMetaData metadata, + final DecoratedKey key, + final Iterator cells, + final int nowInSec) + { + SerializationHelper helper = new SerializationHelper(0, SerializationHelper.Flag.LOCAL); + return UnfilteredRowIterators.filter(toUnfilteredRowIterator(metadata, key, LegacyDeletionInfo.live(), cells, false, helper), nowInSec); + } + + private static LivenessInfo livenessInfo(CFMetaData metadata, LegacyCell cell) + { + return cell.isTombstone() + ? SimpleLivenessInfo.forDeletion(cell.timestamp, cell.localDeletionTime) + : SimpleLivenessInfo.forUpdate(cell.timestamp, cell.ttl, cell.localDeletionTime, metadata); + } + + public static Comparator legacyCellComparator(CFMetaData metadata) + { + return legacyCellComparator(metadata, false); + } + + public static Comparator legacyCellComparator(final CFMetaData metadata, final boolean reversed) + { + final Comparator cellNameComparator = legacyCellNameComparator(metadata, reversed); + return new Comparator() + { + public int compare(LegacyCell cell1, LegacyCell cell2) + { + LegacyCellName c1 = cell1.name; + LegacyCellName c2 = cell2.name; + + int c = cellNameComparator.compare(c1, c2); + if (c != 0) + return c; + + // The actual sorting when the cellname is equal doesn't matter, we just want to make + // sure the cells are not considered equal. + if (cell1.timestamp != cell2.timestamp) + return cell1.timestamp < cell2.timestamp ? -1 : 1; + + if (cell1.localDeletionTime != cell2.localDeletionTime) + return cell1.localDeletionTime < cell2.localDeletionTime ? -1 : 1; + + return cell1.value.compareTo(cell2.value); + } + }; + } + + public static Comparator legacyCellNameComparator(final CFMetaData metadata, final boolean reversed) + { + return new Comparator() + { + public int compare(LegacyCellName c1, LegacyCellName c2) + { + // Compare clustering first + if (c1.clustering == Clustering.STATIC_CLUSTERING) + { + if (c2.clustering != Clustering.STATIC_CLUSTERING) + return -1; + } + else if (c2.clustering == Clustering.STATIC_CLUSTERING) + { + return 1; + } + else + { + int c = metadata.comparator.compare(c1.clustering, c2.clustering); + if (c != 0) + return reversed ? -c : c; + } + + // Note that when reversed, we only care about the clustering being reversed, so it's ok + // not to take reversed into account below. + + // Then check the column name + if (c1.column != c2.column) + { + // A null for the column means it's a row marker + if (c1.column == null) + return -1; + if (c2.column == null) + return 1; + + assert c1.column.isRegular() || c1.column.isStatic(); + assert c2.column.isRegular() || c2.column.isStatic(); + if (c1.column.kind != c2.column.kind) + return c1.column.isStatic() ? -1 : 1; + + AbstractType cmp = metadata.getColumnDefinitionNameComparator(c1.column.kind); + int c = cmp.compare(c1.column.name.bytes, c2.column.name.bytes); + if (c != 0) + return c; + } + + assert (c1.collectionElement == null) == (c2.collectionElement == null); + + if (c1.collectionElement != null) + { + AbstractType colCmp = ((CollectionType)c1.column.type).nameComparator(); + return colCmp.compare(c1.collectionElement, c2.collectionElement); + } + return 0; + } + }; + } + + public static LegacyAtom readLegacyAtom(CFMetaData metadata, DataInput in, boolean readAllAsDynamic) throws IOException + { + while (true) + { + ByteBuffer cellname = ByteBufferUtil.readWithShortLength(in); + if (!cellname.hasRemaining()) + return null; // END_OF_ROW + + try + { + int b = in.readUnsignedByte(); + return (b & RANGE_TOMBSTONE_MASK) != 0 + ? readLegacyRangeTombstoneBody(metadata, in, cellname) + : readLegacyCellBody(metadata, in, cellname, b, SerializationHelper.Flag.LOCAL, readAllAsDynamic); + } + catch (UnknownColumnException e) + { + // We can get there if we read a cell for a dropped column, and ff that is the case, + // then simply ignore the cell is fine. But also not that we ignore if it's the + // system keyspace because for those table we actually remove columns without registering + // them in the dropped columns + assert metadata.ksName.equals(SystemKeyspace.NAME) || metadata.getDroppedColumnDefinition(cellname) != null : e.getMessage(); + } + } + } + + public static LegacyCell readLegacyCell(CFMetaData metadata, DataInput in, SerializationHelper.Flag flag) throws IOException, UnknownColumnException + { + ByteBuffer cellname = ByteBufferUtil.readWithShortLength(in); + int b = in.readUnsignedByte(); + return readLegacyCellBody(metadata, in, cellname, b, flag, false); + } + + public static LegacyCell readLegacyCellBody(CFMetaData metadata, DataInput in, ByteBuffer cellname, int mask, SerializationHelper.Flag flag, boolean readAllAsDynamic) + throws IOException, UnknownColumnException + { + // Note that we want to call decodeCellName only after we've deserialized other parts, since it can throw + // and we want to throw only after having deserialized the full cell. + if ((mask & COUNTER_MASK) != 0) + { + in.readLong(); // timestampOfLastDelete: this has been unused for a long time so we ignore it + long ts = in.readLong(); + ByteBuffer value = ByteBufferUtil.readWithLength(in); + if (flag == SerializationHelper.Flag.FROM_REMOTE || (flag == SerializationHelper.Flag.LOCAL && CounterContext.instance().shouldClearLocal(value))) + value = CounterContext.instance().clearAllLocal(value); + return new LegacyCell(LegacyCell.Kind.COUNTER, decodeCellName(metadata, cellname, readAllAsDynamic), value, ts, LivenessInfo.NO_DELETION_TIME, LivenessInfo.NO_TTL); + } + else if ((mask & EXPIRATION_MASK) != 0) + { + int ttl = in.readInt(); + int expiration = in.readInt(); + long ts = in.readLong(); + ByteBuffer value = ByteBufferUtil.readWithLength(in); + return new LegacyCell(LegacyCell.Kind.EXPIRING, decodeCellName(metadata, cellname, readAllAsDynamic), value, ts, expiration, ttl); + } + else + { + long ts = in.readLong(); + ByteBuffer value = ByteBufferUtil.readWithLength(in); + LegacyCellName name = decodeCellName(metadata, cellname, readAllAsDynamic); + return (mask & COUNTER_UPDATE_MASK) != 0 + ? new LegacyCell(LegacyCell.Kind.COUNTER, name, CounterContext.instance().createLocal(ByteBufferUtil.toLong(value)), ts, LivenessInfo.NO_DELETION_TIME, LivenessInfo.NO_TTL) + : ((mask & DELETION_MASK) == 0 + ? new LegacyCell(LegacyCell.Kind.REGULAR, name, value, ts, LivenessInfo.NO_DELETION_TIME, LivenessInfo.NO_TTL) + : new LegacyCell(LegacyCell.Kind.DELETED, name, ByteBufferUtil.EMPTY_BYTE_BUFFER, ts, ByteBufferUtil.toInt(value), LivenessInfo.NO_TTL)); + } + } + + public static LegacyRangeTombstone readLegacyRangeTombstone(CFMetaData metadata, DataInput in) throws IOException + { + ByteBuffer boundname = ByteBufferUtil.readWithShortLength(in); + in.readUnsignedByte(); + return readLegacyRangeTombstoneBody(metadata, in, boundname); + } + + public static LegacyRangeTombstone readLegacyRangeTombstoneBody(CFMetaData metadata, DataInput in, ByteBuffer boundname) throws IOException + { + LegacyBound min = decodeBound(metadata, boundname, true); + LegacyBound max = decodeBound(metadata, ByteBufferUtil.readWithShortLength(in), false); + DeletionTime dt = DeletionTime.serializer.deserialize(in); + return new LegacyRangeTombstone(min, max, dt); + } + + public static Iterator deserializeCells(final CFMetaData metadata, + final DataInput in, + final SerializationHelper.Flag flag, + final int size) + { + return new AbstractIterator() + { + private int i = 0; + + protected LegacyCell computeNext() + { + if (i >= size) + return endOfData(); + + ++i; + try + { + return readLegacyCell(metadata, in, flag); + } + catch (UnknownColumnException e) + { + // We can get there if we read a cell for a dropped column, and if that is the case, + // then simply ignore the cell is fine. But also not that we ignore if it's the + // system keyspace because for those table we actually remove columns without registering + // them in the dropped columns + if (metadata.ksName.equals(SystemKeyspace.NAME) || metadata.getDroppedColumnDefinition(e.columnName) != null) + return computeNext(); + else + throw new IOError(e); + } + catch (IOException e) + { + throw new IOError(e); + } + } + }; + } + + public static class CellGrouper + { + public final CFMetaData metadata; + private final ReusableRow row; + private final boolean isStatic; + private final SerializationHelper helper; + private Row.Writer writer; + private Clustering clustering; + + private LegacyRangeTombstone rowDeletion; + private LegacyRangeTombstone collectionDeletion; + + public CellGrouper(CFMetaData metadata, SerializationHelper helper) + { + this(metadata, helper, false); + } + + private CellGrouper(CFMetaData metadata, SerializationHelper helper, boolean isStatic) + { + this.metadata = metadata; + this.isStatic = isStatic; + this.helper = helper; + this.row = isStatic ? null : new ReusableRow(metadata.clusteringColumns().size(), metadata.partitionColumns().regulars, false, metadata.isCounter()); + + if (isStatic) + this.writer = StaticRow.builder(metadata.partitionColumns().statics, false, metadata.isCounter()); + } + + public static CellGrouper staticGrouper(CFMetaData metadata, SerializationHelper helper) + { + return new CellGrouper(metadata, helper, true); + } + + public void reset() + { + this.clustering = null; + this.rowDeletion = null; + this.collectionDeletion = null; + + if (!isStatic) + this.writer = row.writer(); + } + + public boolean addAtom(LegacyAtom atom) + { + return atom.isCell() + ? addCell(atom.asCell()) + : addRangeTombstone(atom.asRangeTombstone()); + } + + public boolean addCell(LegacyCell cell) + { + if (isStatic) + { + if (cell.name.clustering != Clustering.STATIC_CLUSTERING) + return false; + } + else if (clustering == null) + { + clustering = cell.name.clustering.takeAlias(); + clustering.writeTo(writer); + } + else if (!clustering.equals(cell.name.clustering)) + { + return false; + } + + // Ignore shadowed cells + if (rowDeletion != null && rowDeletion.deletionTime.deletes(cell.timestamp)) + return true; + + LivenessInfo info = livenessInfo(metadata, cell); + + ColumnDefinition column = cell.name.column; + if (column == null) + { + // It's the row marker + assert !cell.value.hasRemaining(); + helper.writePartitionKeyLivenessInfo(writer, info.timestamp(), info.ttl(), info.localDeletionTime()); + } + else + { + if (collectionDeletion != null && collectionDeletion.start.collectionName.name.equals(column.name) && collectionDeletion.deletionTime.deletes(cell.timestamp)) + return true; + + if (helper.includes(column)) + { + CellPath path = null; + if (column.isComplex()) + { + // Recalling startOfComplexColumn for every cell is a big inefficient, but it's ok in practice + // and it's simpler. And since 1) this only matter for super column selection in thrift in + // practice and 2) is only used during upgrade, it's probably worth keeping things simple. + helper.startOfComplexColumn(column); + path = cell.name.collectionElement == null ? null : CellPath.create(cell.name.collectionElement); + } + helper.writeCell(writer, column, cell.isCounter(), cell.value, info.timestamp(), info.localDeletionTime(), info.ttl(), path); + if (column.isComplex()) + { + helper.endOfComplexColumn(column); + } + } + } + return true; + } + + public boolean addRangeTombstone(LegacyRangeTombstone tombstone) + { + if (tombstone.isRowDeletion(metadata)) + { + // If we're already within a row, it can't be the same one + if (clustering != null) + return false; + + clustering = tombstone.start.getAsClustering(metadata).takeAlias(); + clustering.writeTo(writer); + writer.writeRowDeletion(tombstone.deletionTime); + rowDeletion = tombstone; + return true; + } + + if (tombstone.isCollectionTombstone()) + { + if (clustering == null) + { + clustering = tombstone.start.getAsClustering(metadata).takeAlias(); + clustering.writeTo(writer); + } + else if (!clustering.equals(tombstone.start.getAsClustering(metadata))) + { + return false; + } + + writer.writeComplexDeletion(tombstone.start.collectionName, tombstone.deletionTime); + if (rowDeletion == null || tombstone.deletionTime.supersedes(rowDeletion.deletionTime)) + collectionDeletion = tombstone; + return true; + } + return false; + } + + public Row getRow() + { + writer.endOfRow(); + return isStatic ? ((StaticRow.Builder)writer).build() : row; + } + } + + public static class LegacyCellName + { + public final Clustering clustering; + public final ColumnDefinition column; + public final ByteBuffer collectionElement; + + private LegacyCellName(Clustering clustering, ColumnDefinition column, ByteBuffer collectionElement) + { + this.clustering = clustering; + this.column = column; + this.collectionElement = collectionElement; + } + + public ByteBuffer encode(CFMetaData metadata) + { + return encodeCellName(metadata, clustering, column == null ? ByteBufferUtil.EMPTY_BYTE_BUFFER : column.name.bytes, collectionElement); + } + + public ByteBuffer superColumnSubName() + { + assert collectionElement != null; + return collectionElement; + } + + public ByteBuffer superColumnName() + { + return clustering.get(0); + } + + @Override + public String toString() + { + StringBuilder sb = new StringBuilder(); + for (int i = 0; i < clustering.size(); i++) + sb.append(i > 0 ? ":" : "").append(clustering.get(i) == null ? "null" : ByteBufferUtil.bytesToHex(clustering.get(i))); + return String.format("Cellname(clustering=%s, column=%s, collElt=%s)", sb.toString(), column == null ? "null" : column.name, collectionElement == null ? "null" : ByteBufferUtil.bytesToHex(collectionElement)); + } + } + + public static class LegacyBound + { + public static final LegacyBound BOTTOM = new LegacyBound(Slice.Bound.BOTTOM, false, null); + public static final LegacyBound TOP = new LegacyBound(Slice.Bound.TOP, false, null); + + public final Slice.Bound bound; + public final boolean isStatic; + public final ColumnDefinition collectionName; + + private LegacyBound(Slice.Bound bound, boolean isStatic, ColumnDefinition collectionName) + { + this.bound = bound; + this.isStatic = isStatic; + this.collectionName = collectionName; + } + + public Clustering getAsClustering(CFMetaData metadata) + { + assert bound.size() == metadata.comparator.size(); + ByteBuffer[] values = new ByteBuffer[bound.size()]; + for (int i = 0; i < bound.size(); i++) + values[i] = bound.get(i); + return new SimpleClustering(values); + } + + @Override + public String toString() + { + StringBuilder sb = new StringBuilder(); + sb.append(bound.kind()).append('('); + for (int i = 0; i < bound.size(); i++) + sb.append(i > 0 ? ":" : "").append(bound.get(i) == null ? "null" : ByteBufferUtil.bytesToHex(bound.get(i))); + sb.append(')'); + return String.format("Bound(%s, collection=%s)", sb.toString(), collectionName == null ? "null" : collectionName.name); + } + } + + public interface LegacyAtom extends Clusterable + { + public boolean isCell(); + + public ClusteringPrefix clustering(); + public boolean isStatic(); + + public LegacyCell asCell(); + public LegacyRangeTombstone asRangeTombstone(); + } + + /** + * A legacy cell. + *

+ * This is used as a temporary object to facilitate dealing with the legacy format, this + * is not meant to be optimal. + */ + public static class LegacyCell implements LegacyAtom + { + public enum Kind { REGULAR, EXPIRING, DELETED, COUNTER } + + public final Kind kind; + + public final LegacyCellName name; + public final ByteBuffer value; + + public final long timestamp; + public final int localDeletionTime; + public final int ttl; + + private LegacyCell(Kind kind, LegacyCellName name, ByteBuffer value, long timestamp, int localDeletionTime, int ttl) + { + this.kind = kind; + this.name = name; + this.value = value; + this.timestamp = timestamp; + this.localDeletionTime = localDeletionTime; + this.ttl = ttl; + } + + public static LegacyCell regular(CFMetaData metadata, ByteBuffer superColumnName, ByteBuffer name, ByteBuffer value, long timestamp) + throws UnknownColumnException + { + return new LegacyCell(Kind.REGULAR, decodeCellName(metadata, superColumnName, name), value, timestamp, LivenessInfo.NO_DELETION_TIME, LivenessInfo.NO_TTL); + } + + public static LegacyCell expiring(CFMetaData metadata, ByteBuffer superColumnName, ByteBuffer name, ByteBuffer value, long timestamp, int ttl, int nowInSec) + throws UnknownColumnException + { + return new LegacyCell(Kind.EXPIRING, decodeCellName(metadata, superColumnName, name), value, timestamp, nowInSec, ttl); + } + + public static LegacyCell tombstone(CFMetaData metadata, ByteBuffer superColumnName, ByteBuffer name, long timestamp, int nowInSec) + throws UnknownColumnException + { + return new LegacyCell(Kind.DELETED, decodeCellName(metadata, superColumnName, name), ByteBufferUtil.EMPTY_BYTE_BUFFER, timestamp, nowInSec, LivenessInfo.NO_TTL); + } + + public static LegacyCell counter(CFMetaData metadata, ByteBuffer superColumnName, ByteBuffer name, long value) + throws UnknownColumnException + { + // See UpdateParameters.addCounter() for more details on this + ByteBuffer counterValue = CounterContext.instance().createLocal(value); + return counter(decodeCellName(metadata, superColumnName, name), counterValue); + } + + public static LegacyCell counter(LegacyCellName name, ByteBuffer value) + { + return new LegacyCell(Kind.COUNTER, name, value, FBUtilities.timestampMicros(), LivenessInfo.NO_DELETION_TIME, LivenessInfo.NO_TTL); + } + + public ClusteringPrefix clustering() + { + return name.clustering; + } + + public boolean isStatic() + { + return name.clustering == Clustering.STATIC_CLUSTERING; + } + + public boolean isCell() + { + return true; + } + + public LegacyCell asCell() + { + return this; + } + + public LegacyRangeTombstone asRangeTombstone() + { + throw new UnsupportedOperationException(); + } + + public boolean isCounter() + { + return kind == Kind.COUNTER; + } + + public boolean isExpiring() + { + return kind == Kind.EXPIRING; + } + + public boolean isTombstone() + { + return kind == Kind.DELETED; + } + + public boolean isLive(int nowInSec) + { + if (isTombstone()) + return false; + + if (isExpiring()) + return nowInSec < localDeletionTime; + + return true; + } + + @Override + public String toString() + { + return String.format("LegacyCell(%s, name=%s, v=%s, ts=%s, ldt=%s, ttl=%s)", kind, name, ByteBufferUtil.bytesToHex(value), timestamp, localDeletionTime, ttl); + } + } + + /** + * A legacy range tombstone. + *

+ * This is used as a temporary object to facilitate dealing with the legacy format, this + * is not meant to be optimal. + */ + public static class LegacyRangeTombstone implements LegacyAtom + { + public final LegacyBound start; + public final LegacyBound stop; + public final DeletionTime deletionTime; + + // Do not use directly, use create() instead. + private LegacyRangeTombstone(LegacyBound start, LegacyBound stop, DeletionTime deletionTime) + { + // Because of the way RangeTombstoneList work, we can have a tombstone where only one of + // the bound has a collectionName. That happens if we have a big tombstone A (spanning one + // or multiple rows) and a collection tombstone B. In that case, RangeTombstoneList will + // split this into 3 RTs: the first one from the beginning of A to the beginning of B, + // then B, then a third one from the end of B to the end of A. To make this simpler, if + // we detect that case we transform the 1st and 3rd tombstone so they don't end in the middle + // of a row (which is still correct). + if ((start.collectionName == null) != (stop.collectionName == null)) + { + if (start.collectionName == null) + stop = new LegacyBound(stop.bound, stop.isStatic, null); + else + start = new LegacyBound(start.bound, start.isStatic, null); + } + else if (!Objects.equals(start.collectionName, stop.collectionName)) + { + // We're in the similar but slightly more complex case where on top of the big tombstone + // A, we have 2 (or more) collection tombstones B and C within A. So we also end up with + // a tombstone that goes between the end of B and the start of C. + start = new LegacyBound(start.bound, start.isStatic, null); + stop = new LegacyBound(stop.bound, stop.isStatic, null); + } + + this.start = start; + this.stop = stop; + this.deletionTime = deletionTime; + } + + public ClusteringPrefix clustering() + { + return start.bound; + } + + public boolean isCell() + { + return false; + } + + public boolean isStatic() + { + return start.isStatic; + } + + public LegacyCell asCell() + { + throw new UnsupportedOperationException(); + } + + public LegacyRangeTombstone asRangeTombstone() + { + return this; + } + + public boolean isCollectionTombstone() + { + return start.collectionName != null; + } + + public boolean isRowDeletion(CFMetaData metadata) + { + if (start.collectionName != null + || stop.collectionName != null + || start.bound.size() != metadata.comparator.size() + || stop.bound.size() != metadata.comparator.size()) + return false; + + for (int i = 0; i < start.bound.size(); i++) + if (!Objects.equals(start.bound.get(i), stop.bound.get(i))) + return false; + return true; + } + + @Override + public String toString() + { + return String.format("RT(%s-%s, %s)", start, stop, deletionTime); + } + } + + public static class LegacyDeletionInfo + { + public static final Serializer serializer = new Serializer(); + + public final DeletionInfo deletionInfo; + private final List inRowTombstones; + + private LegacyDeletionInfo(DeletionInfo deletionInfo, List inRowTombstones) + { + this.deletionInfo = deletionInfo; + this.inRowTombstones = inRowTombstones; + } + + public static LegacyDeletionInfo from(DeletionInfo info) + { + return new LegacyDeletionInfo(info, Collections.emptyList()); + } + + public static LegacyDeletionInfo live() + { + return from(DeletionInfo.live()); + } + + public Iterator inRowRangeTombstones() + { + return inRowTombstones.iterator(); + } + + public static class Serializer + { + public void serialize(CFMetaData metadata, LegacyDeletionInfo info, DataOutputPlus out, int version) throws IOException + { + throw new UnsupportedOperationException(); + //DeletionTime.serializer.serialize(info.topLevel, out); + //rtlSerializer.serialize(info.ranges, out, version); + } + + public LegacyDeletionInfo deserialize(CFMetaData metadata, DataInput in, int version) throws IOException + { + DeletionTime topLevel = DeletionTime.serializer.deserialize(in); + + int rangeCount = in.readInt(); + if (rangeCount == 0) + return from(new DeletionInfo(topLevel)); + + RangeTombstoneList ranges = new RangeTombstoneList(metadata.comparator, rangeCount); + List inRowTombsones = new ArrayList<>(); + for (int i = 0; i < rangeCount; i++) + { + LegacyBound start = decodeBound(metadata, ByteBufferUtil.readWithShortLength(in), true); + LegacyBound end = decodeBound(metadata, ByteBufferUtil.readWithShortLength(in), false); + int delTime = in.readInt(); + long markedAt = in.readLong(); + + LegacyRangeTombstone tombstone = new LegacyRangeTombstone(start, end, new SimpleDeletionTime(markedAt, delTime)); + if (tombstone.isCollectionTombstone() || tombstone.isRowDeletion(metadata)) + inRowTombsones.add(tombstone); + else + ranges.add(start.bound, end.bound, markedAt, delTime); + } + return new LegacyDeletionInfo(new DeletionInfo(topLevel, ranges), inRowTombsones); + } + + public long serializedSize(CFMetaData metadata, LegacyDeletionInfo info, TypeSizes typeSizes, int version) + { + throw new UnsupportedOperationException(); + //long size = DeletionTime.serializer.serializedSize(info.topLevel, typeSizes); + //return size + rtlSerializer.serializedSize(info.ranges, typeSizes, version); + } + } + } + + public static class TombstoneTracker + { + private final CFMetaData metadata; + private final DeletionTime partitionDeletion; + private final List openTombstones = new ArrayList<>(); + + public TombstoneTracker(CFMetaData metadata, DeletionTime partitionDeletion) + { + this.metadata = metadata; + this.partitionDeletion = partitionDeletion; + } + + public void update(LegacyAtom atom) + { + if (atom.isCell()) + { + if (openTombstones.isEmpty()) + return; + + Iterator iter = openTombstones.iterator(); + while (iter.hasNext()) + { + LegacyRangeTombstone tombstone = iter.next(); + if (metadata.comparator.compare(atom, tombstone.stop.bound) >= 0) + iter.remove(); + } + } + + LegacyRangeTombstone tombstone = atom.asRangeTombstone(); + if (tombstone.deletionTime.supersedes(partitionDeletion) && !tombstone.isRowDeletion(metadata) && !tombstone.isCollectionTombstone()) + openTombstones.add(tombstone); + } + + public boolean isShadowed(LegacyAtom atom) + { + long timestamp = atom.isCell() ? atom.asCell().timestamp : atom.asRangeTombstone().deletionTime.markedForDeleteAt(); + + if (partitionDeletion.deletes(timestamp)) + return true; + + for (LegacyRangeTombstone tombstone : openTombstones) + { + if (tombstone.deletionTime.deletes(timestamp)) + return true; + } + + return false; + } + } +} diff --git a/src/java/org/apache/cassandra/db/LivenessInfo.java b/src/java/org/apache/cassandra/db/LivenessInfo.java new file mode 100644 index 0000000000..89971d18d1 --- /dev/null +++ b/src/java/org/apache/cassandra/db/LivenessInfo.java @@ -0,0 +1,186 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.cassandra.db; + +import java.security.MessageDigest; + +import org.apache.cassandra.db.*; + +/** + * Groups the informations necessary to decide the liveness of a given piece of + * column data. + *

+ * In practice, a {@code LivenessInfo} groups 3 informations: + * 1) the data timestamp. It is sometimes allowed for a given piece of data to have + * no timestamp (for {@link Row#partitionKeyLivenessInfo} more precisely), but if that + * is the case it means the data has no liveness info at all. + * 2) the data ttl if relevant. + * 2) the data local deletion time if relevant (that is, if either the data has a ttl or is deleted). + */ +public interface LivenessInfo extends Aliasable +{ + public static final long NO_TIMESTAMP = Long.MIN_VALUE; + public static final int NO_TTL = 0; + public static final int NO_DELETION_TIME = Integer.MAX_VALUE; + + public static final LivenessInfo NONE = new SimpleLivenessInfo(NO_TIMESTAMP, NO_TTL, NO_DELETION_TIME); + + /** + * The timestamp at which the data was inserted or {@link NO_TIMESTAMP} + * if it has no timestamp (which may or may not be allowed). + * + * @return the liveness info timestamp. + */ + public long timestamp(); + + /** + * Whether this liveness info has a timestamp or not. + *

+ * Note that if this return {@code false}, then both {@link #hasTTL} and + * {@link #hasLocalDeletionTime} must return {@code false} too. + * + * @return whether this liveness info has a timestamp or not. + */ + public boolean hasTimestamp(); + + /** + * The ttl (if any) on the data or {@link NO_TTL} if the data is not + * expiring. + * + * Please note that this value is the TTL that was set originally and is thus not + * changing. If you want to figure out how much time the data has before it expires, + * then you should use {@link #remainingTTL}. + */ + public int ttl(); + + /** + * Whether this liveness info has a TTL or not. + * + * @return whether this liveness info has a TTL or not. + */ + public boolean hasTTL(); + + /** + * The deletion time (in seconds) on the data if applicable ({@link NO_DELETION} + * otherwise). + * + * There is 3 cases in practice: + * 1) the data is neither deleted nor expiring: it then has neither {@code ttl()} + * nor {@code localDeletionTime()}. + * 2) the data is expiring/expired: it then has both a {@code ttl()} and a + * {@code localDeletionTime()}. Whether it's still live or is expired depends + * on the {@code localDeletionTime()}. + * 3) the data is deleted: it has no {@code ttl()} but has a + * {@code localDeletionTime()}. + */ + public int localDeletionTime(); + + /** + * Whether this liveness info has a local deletion time or not. + * + * @return whether this liveness info has a local deletion time or not. + */ + public boolean hasLocalDeletionTime(); + + /** + * The actual remaining time to live (in seconds) for the data this is + * the liveness information of. + * + * {@code #ttl} returns the initial TTL sets on the piece of data while this + * method computes how much time the data actually has to live given the + * current time. + * + * @param nowInSec the current time in seconds. + * @return the remaining time to live (in seconds) the data has, or + * {@code -1} if the data is either expired or not expiring. + */ + public int remainingTTL(int nowInSec); + + /** + * Checks whether a given piece of data is live given the current time. + * + * @param nowInSec the current time in seconds. + * @return whether the data having this liveness info is live or not. + */ + public boolean isLive(int nowInSec); + + /** + * Adds this liveness information to the provided digest. + * + * @param digest the digest to add this liveness information to. + */ + public void digest(MessageDigest digest); + + /** + * Validate the data contained by this liveness information. + * + * @throws MarshalException if some of the data is corrupted. + */ + public void validate(); + + /** + * The size of the (useful) data this liveness information contains. + * + * @return the size of the data this liveness information contains. + */ + public int dataSize(); + + /** + * Whether this liveness information supersedes another one (that is + * whether is has a greater timestamp than the other or not). + * + * @param other the {@code LivenessInfo} to compare this info to. + * + * @return whether this {@code LivenessInfo} supersedes {@code other}. + */ + public boolean supersedes(LivenessInfo other); + + /** + * Returns the result of merging this info to another one (that is, it + * return this info if it supersedes the other one, or the other one + * otherwise). + */ + public LivenessInfo mergeWith(LivenessInfo other); + + /** + * Whether this liveness information can be purged. + *

+ * A liveness info can be purged if it is not live and hasn't been so + * for longer than gcGrace (or more precisely, it's local deletion time + * is smaller than gcBefore, which is itself "now - gcGrace"). + * + * @param maxPurgeableTimestamp the biggest timestamp that can be purged. + * A liveness info will not be considered purgeable if its timestamp is + * greater than this value, even if it mets the other criteria for purging. + * @param gcBefore the local deletion time before which deleted/expired + * liveness info can be purged. + * + * @return whether this liveness information can be purged. + */ + public boolean isPurgeable(long maxPurgeableTimestamp, int gcBefore); + + /** + * Returns a copy of this liveness info updated with the provided timestamp. + * + * @param newTimestamp the timestamp for the returned info. + * @return if this liveness info has a timestamp, a copy of it with {@code newTimestamp} + * as timestamp. If it has no timestamp however, this liveness info is returned + * unchanged. + */ + public LivenessInfo withUpdatedTimestamp(long newTimestamp); +} diff --git a/src/java/org/apache/cassandra/db/LivenessInfoArray.java b/src/java/org/apache/cassandra/db/LivenessInfoArray.java new file mode 100644 index 0000000000..24026d8e47 --- /dev/null +++ b/src/java/org/apache/cassandra/db/LivenessInfoArray.java @@ -0,0 +1,174 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.cassandra.db; + +import java.util.Arrays; + +import org.apache.cassandra.utils.ObjectSizes; + +/** + * Utility class to store an array of liveness info efficiently. + */ +public class LivenessInfoArray +{ + private long[] timestamps; + private int[] delTimesAndTTLs; + + public LivenessInfoArray(int initialCapacity) + { + this.timestamps = new long[initialCapacity]; + this.delTimesAndTTLs = new int[initialCapacity * 2]; + clear(); + } + + public void clear(int i) + { + timestamps[i] = LivenessInfo.NO_TIMESTAMP; + delTimesAndTTLs[2 * i] = LivenessInfo.NO_DELETION_TIME; + delTimesAndTTLs[2 * i + 1] = LivenessInfo.NO_TTL; + } + + public void set(int i, LivenessInfo info) + { + set(i, info.timestamp(), info.ttl(), info.localDeletionTime()); + } + + public void set(int i, long timestamp, int ttl, int localDeletionTime) + { + this.timestamps[i] = timestamp; + this.delTimesAndTTLs[2 * i] = localDeletionTime; + this.delTimesAndTTLs[2 * i + 1] = ttl; + } + + public long timestamp(int i) + { + return timestamps[i]; + } + + public int localDeletionTime(int i) + { + return delTimesAndTTLs[2 * i]; + } + + public int ttl(int i) + { + return delTimesAndTTLs[2 * i + 1]; + } + + public boolean isLive(int i, int nowInSec) + { + // See AbstractLivenessInfo.isLive(). + return localDeletionTime(i) == LivenessInfo.NO_DELETION_TIME + || (ttl(i) != LivenessInfo.NO_TTL && nowInSec < localDeletionTime(i)); + } + + public int size() + { + return timestamps.length; + } + + public void resize(int newSize) + { + int prevSize = size(); + + timestamps = Arrays.copyOf(timestamps, newSize); + delTimesAndTTLs = Arrays.copyOf(delTimesAndTTLs, newSize * 2); + + clear(prevSize, newSize); + } + + public void swap(int i, int j) + { + long ts = timestamps[j]; + int ldt = delTimesAndTTLs[2 * j]; + int ttl = delTimesAndTTLs[2 * j + 1]; + + move(i, j); + + timestamps[i] = ts; + delTimesAndTTLs[2 * i] = ldt; + delTimesAndTTLs[2 * i + 1] = ttl; + } + + public void move(int i, int j) + { + timestamps[j] = timestamps[i]; + delTimesAndTTLs[2 * j] = delTimesAndTTLs[2 * i]; + delTimesAndTTLs[2 * j + 1] = delTimesAndTTLs[2 * i + 1]; + } + + public void clear() + { + clear(0, size()); + } + + private void clear(int from, int to) + { + Arrays.fill(timestamps, from, to, LivenessInfo.NO_TIMESTAMP); + for (int i = from; i < to; i++) + { + delTimesAndTTLs[2 * i] = LivenessInfo.NO_DELETION_TIME; + delTimesAndTTLs[2 * i + 1] = LivenessInfo.NO_TTL; + } + } + + public int dataSize() + { + return 16 * size(); + } + + public long unsharedHeapSize() + { + return ObjectSizes.sizeOfArray(timestamps) + + ObjectSizes.sizeOfArray(delTimesAndTTLs); + } + + public static Cursor newCursor() + { + return new Cursor(); + } + + public static class Cursor extends AbstractLivenessInfo + { + private LivenessInfoArray array; + private int i; + + public Cursor setTo(LivenessInfoArray array, int i) + { + this.array = array; + this.i = i; + return this; + } + + public long timestamp() + { + return array.timestamps[i]; + } + + public int localDeletionTime() + { + return array.delTimesAndTTLs[2 * i]; + } + + public int ttl() + { + return array.delTimesAndTTLs[2 * i + 1]; + } + } +} + diff --git a/src/java/org/apache/cassandra/db/Memtable.java b/src/java/org/apache/cassandra/db/Memtable.java index ccf92be84d..e82c35e64e 100644 --- a/src/java/org/apache/cassandra/db/Memtable.java +++ b/src/java/org/apache/cassandra/db/Memtable.java @@ -18,30 +18,33 @@ package org.apache.cassandra.db; import java.io.File; -import java.util.AbstractMap; -import java.util.Iterator; -import java.util.Map; +import java.util.*; import java.util.concurrent.ConcurrentNavigableMap; import java.util.concurrent.ConcurrentSkipListMap; +import java.util.concurrent.ConcurrentSkipListSet; import java.util.concurrent.TimeUnit; +import java.util.concurrent.atomic.AtomicBoolean; import java.util.concurrent.atomic.AtomicLong; import java.util.concurrent.atomic.AtomicReference; import com.google.common.annotations.VisibleForTesting; - -import org.apache.cassandra.config.CFMetaData; -import org.apache.cassandra.io.sstable.Descriptor; -import org.apache.cassandra.io.sstable.format.SSTableReader; -import org.apache.cassandra.io.sstable.format.SSTableWriter; import org.slf4j.Logger; import org.slf4j.LoggerFactory; +import org.apache.cassandra.config.CFMetaData; +import org.apache.cassandra.config.ColumnDefinition; import org.apache.cassandra.config.DatabaseDescriptor; +import org.apache.cassandra.db.rows.*; import org.apache.cassandra.db.commitlog.CommitLog; import org.apache.cassandra.db.commitlog.ReplayPosition; -import org.apache.cassandra.db.composites.CellNameType; import org.apache.cassandra.db.index.SecondaryIndexManager; import org.apache.cassandra.dht.Murmur3Partitioner.LongToken; +import org.apache.cassandra.db.filter.*; +import org.apache.cassandra.db.partitions.*; +import org.apache.cassandra.dht.*; +import org.apache.cassandra.io.sstable.Descriptor; +import org.apache.cassandra.io.sstable.format.SSTableReader; +import org.apache.cassandra.io.sstable.format.SSTableWriter; import org.apache.cassandra.io.sstable.metadata.MetadataCollector; import org.apache.cassandra.io.util.DiskAwareRunnable; import org.apache.cassandra.service.ActiveRepairService; @@ -79,10 +82,10 @@ public class Memtable implements Comparable } } - // We index the memtable by RowPosition only for the purpose of being able + // We index the memtable by PartitionPosition only for the purpose of being able // to select key range using Token.KeyBound. However put() ensures that we // actually only store DecoratedKey. - private final ConcurrentNavigableMap rows = new ConcurrentSkipListMap<>(); + private final ConcurrentNavigableMap partitions = new ConcurrentSkipListMap<>(); public final ColumnFamilyStore cfs; private final long creationTime = System.currentTimeMillis(); private final long creationNano = System.nanoTime(); @@ -90,7 +93,10 @@ public class Memtable implements Comparable // Record the comparator of the CFS at the creation of the memtable. This // is only used when a user update the CF comparator, to know if the // memtable was created with the new or old comparator. - public final CellNameType initialComparator; + public final ClusteringComparator initialComparator; + + private final ColumnsCollector columnsCollector; + private final StatsCollector statsCollector = new StatsCollector(); public Memtable(ColumnFamilyStore cfs) { @@ -98,6 +104,7 @@ public class Memtable implements Comparable this.allocator = MEMORY_POOL.newAllocator(); this.initialComparator = cfs.metadata.comparator; this.cfs.scheduleFlush(); + this.columnsCollector = new ColumnsCollector(cfs.metadata.partitionColumns()); } // ONLY to be used for testing, to create a mock Memtable @@ -107,6 +114,7 @@ public class Memtable implements Comparable this.initialComparator = metadata.comparator; this.cfs = null; this.allocator = null; + this.columnsCollector = new ColumnsCollector(metadata.partitionColumns()); } public MemtableAllocator getAllocator() @@ -175,7 +183,7 @@ public class Memtable implements Comparable public boolean isClean() { - return rows.isEmpty(); + return partitions.isEmpty(); } public boolean isCleanAfter(ReplayPosition position) @@ -198,23 +206,23 @@ public class Memtable implements Comparable * * replayPosition should only be null if this is a secondary index, in which case it is *expected* to be null */ - long put(DecoratedKey key, ColumnFamily cf, SecondaryIndexManager.Updater indexer, OpOrder.Group opGroup) + long put(PartitionUpdate update, SecondaryIndexManager.Updater indexer, OpOrder.Group opGroup) { - AtomicBTreeColumns previous = rows.get(key); + AtomicBTreePartition previous = partitions.get(update.partitionKey()); long initialSize = 0; if (previous == null) { - AtomicBTreeColumns empty = cf.cloneMeShallow(AtomicBTreeColumns.factory, false); - final DecoratedKey cloneKey = allocator.clone(key, opGroup); + final DecoratedKey cloneKey = allocator.clone(update.partitionKey(), opGroup); + AtomicBTreePartition empty = new AtomicBTreePartition(cfs.metadata, cloneKey, allocator); // We'll add the columns later. This avoids wasting works if we get beaten in the putIfAbsent - previous = rows.putIfAbsent(cloneKey, empty); + previous = partitions.putIfAbsent(cloneKey, empty); if (previous == null) { previous = empty; // allocate the row overhead after the fact; this saves over allocating and having to free after, but // means we can overshoot our declared limit. - int overhead = (int) (key.getToken().getHeapSize() + ROW_OVERHEAD_HEAP_SIZE); + int overhead = (int) (cloneKey.getToken().getHeapSize() + ROW_OVERHEAD_HEAP_SIZE); allocator.onHeap().allocate(overhead, opGroup); initialSize = 8; } @@ -224,28 +232,17 @@ public class Memtable implements Comparable } } - final Pair pair = previous.addAllWithSizeDelta(cf, allocator, opGroup, indexer); - liveDataSize.addAndGet(initialSize + pair.left); - currentOperations.addAndGet(cf.getColumnCount() + (cf.isMarkedForDelete() ? 1 : 0) + cf.deletionInfo().rangeCount()); - return pair.right; - } - - // for debugging - public String contents() - { - StringBuilder builder = new StringBuilder(); - builder.append("{"); - for (Map.Entry entry : rows.entrySet()) - { - builder.append(entry.getKey()).append(": ").append(entry.getValue()).append(", "); - } - builder.append("}"); - return builder.toString(); + long[] pair = previous.addAllWithSizeDelta(update, opGroup, indexer); + liveDataSize.addAndGet(initialSize + pair[0]); + columnsCollector.update(update.columns()); + statsCollector.update(update.stats()); + currentOperations.addAndGet(update.operationCount()); + return pair[1]; } public int partitionCount() { - return rows.size(); + return partitions.size(); } public FlushRunnable flushRunnable() @@ -259,55 +256,53 @@ public class Memtable implements Comparable cfs.name, hashCode(), liveDataSize, currentOperations, 100 * allocator.onHeap().ownershipRatio(), 100 * allocator.offHeap().ownershipRatio()); } - /** - * @param startWith Include data in the result from and including this key and to the end of the memtable - * @return An iterator of entries with the data from the start key - */ - public Iterator> getEntryIterator(final RowPosition startWith, final RowPosition stopAt) + public UnfilteredPartitionIterator makePartitionIterator(final ColumnFilter columnFilter, final DataRange dataRange, final boolean isForThrift) { - return new Iterator>() - { - private Iterator> iter = stopAt.isMinimum() - ? rows.tailMap(startWith).entrySet().iterator() - : rows.subMap(startWith, true, stopAt, true).entrySet().iterator(); + AbstractBounds keyRange = dataRange.keyRange(); - private Map.Entry currentEntry; + boolean startIsMin = keyRange.left.isMinimum(); + boolean stopIsMin = keyRange.right.isMinimum(); + + boolean isBound = keyRange instanceof Bounds; + boolean includeStart = isBound || keyRange instanceof IncludingExcludingBounds; + boolean includeStop = isBound || keyRange instanceof Range; + Map subMap; + if (startIsMin) + subMap = stopIsMin ? partitions : partitions.headMap(keyRange.right, includeStop); + else + subMap = stopIsMin + ? partitions.tailMap(keyRange.left, includeStart) + : partitions.subMap(keyRange.left, includeStart, keyRange.right, includeStop); + + final Iterator> iter = subMap.entrySet().iterator(); + + return new AbstractUnfilteredPartitionIterator() + { + public boolean isForThrift() + { + return isForThrift; + } public boolean hasNext() { return iter.hasNext(); } - public Map.Entry next() + public UnfilteredRowIterator next() { - Map.Entry entry = iter.next(); + Map.Entry entry = iter.next(); // Actual stored key should be true DecoratedKey assert entry.getKey() instanceof DecoratedKey; - if (MEMORY_POOL.needToCopyOnHeap()) - { - DecoratedKey key = (DecoratedKey) entry.getKey(); - key = new BufferDecoratedKey(key.getToken(), HeapAllocator.instance.clone(key.getKey())); - ColumnFamily cells = ArrayBackedSortedColumns.localCopy(entry.getValue(), HeapAllocator.instance); - entry = new AbstractMap.SimpleImmutableEntry<>(key, cells); - } - // Store the reference to the current entry so that remove() can update the current size. - currentEntry = entry; - // Object cast is required since otherwise we can't turn RowPosition into DecoratedKey - return (Map.Entry) entry; - } - - public void remove() - { - iter.remove(); - liveDataSize.addAndGet(-currentEntry.getValue().dataSize()); - currentEntry = null; + DecoratedKey key = (DecoratedKey)entry.getKey(); + ClusteringIndexFilter filter = dataRange.clusteringIndexFilter(key); + return filter.getUnfilteredRowIterator(columnFilter, entry.getValue()); } }; } - public ColumnFamily getColumnFamily(DecoratedKey key) + public Partition getPartition(DecoratedKey key) { - return rows.get(key); + return partitions.get(key); } public long creationTime() @@ -320,12 +315,14 @@ public class Memtable implements Comparable private final ReplayPosition context; private final long estimatedSize; + private final boolean isBatchLogTable; + FlushRunnable(ReplayPosition context) { this.context = context; long keySize = 0; - for (RowPosition key : rows.keySet()) + for (PartitionPosition key : partitions.keySet()) { // make sure we don't write non-sensical keys assert key instanceof DecoratedKey; @@ -335,6 +332,8 @@ public class Memtable implements Comparable + keySize // keys in data file + liveDataSize.get()) // data * 1.2); // bloom filter and row index overhead + + this.isBatchLogTable = cfs.name.equals(SystemKeyspace.BATCHLOG) && cfs.keyspace.getName().equals(SystemKeyspace.NAME); } public long getExpectedWriteSize() @@ -363,32 +362,32 @@ public class Memtable implements Comparable SSTableReader ssTable; // errors when creating the writer that may leave empty temp files. - try (SSTableWriter writer = createFlushWriter(cfs.getTempSSTablePath(sstableDirectory))) + try (SSTableWriter writer = createFlushWriter(cfs.getTempSSTablePath(sstableDirectory), columnsCollector.get(), statsCollector.get())) { boolean trackContention = logger.isDebugEnabled(); int heavilyContendedRowCount = 0; // (we can't clear out the map as-we-go to free up memory, // since the memtable is being used for queries in the "pending flush" category) - for (Map.Entry entry : rows.entrySet()) + for (AtomicBTreePartition partition : partitions.values()) { - AtomicBTreeColumns cf = entry.getValue(); + // Each batchlog partition is a separate entry in the log. And for an entry, we only do 2 + // operations: 1) we insert the entry and 2) we delete it. Further, BL data is strictly local, + // we don't need to preserve tombstones for repair. So if both operation are in this + // memtable (which will almost always be the case if there is no ongoing failure), we can + // just skip the entry (CASSANDRA-4667). + if (isBatchLogTable && !partition.partitionLevelDeletion().isLive() && partition.hasRows()) + continue; - if (cf.isMarkedForDelete() && cf.hasColumns()) - { - // When every node is up, there's no reason to write batchlog data out to sstables - // (which in turn incurs cost like compaction) since the BL write + delete cancel each other out, - // and BL data is strictly local, so we don't need to preserve tombstones for repair. - // If we have a data row + row level tombstone, then writing it is effectively an expensive no-op so we skip it. - // See CASSANDRA-4667. - if (cfs.name.equals(SystemKeyspace.BATCHLOG) && cfs.keyspace.getName().equals(SystemKeyspace.NAME)) - continue; - } - - if (trackContention && cf.usePessimisticLocking()) + if (trackContention && partition.usePessimisticLocking()) heavilyContendedRowCount++; - if (!cf.isEmpty()) - writer.append((DecoratedKey)entry.getKey(), cf); + if (!partition.isEmpty()) + { + try (UnfilteredRowIterator iter = partition.unfilteredIterator()) + { + writer.append(iter); + } + } } if (writer.getFilePointer() > 0) @@ -406,17 +405,24 @@ public class Memtable implements Comparable } if (heavilyContendedRowCount > 0) - logger.debug(String.format("High update contention in %d/%d partitions of %s ", heavilyContendedRowCount, rows.size(), Memtable.this.toString())); + logger.debug(String.format("High update contention in %d/%d partitions of %s ", heavilyContendedRowCount, partitions.size(), Memtable.this.toString())); return ssTable; } } - public SSTableWriter createFlushWriter(String filename) + public SSTableWriter createFlushWriter(String filename, + PartitionColumns columns, + RowStats stats) { MetadataCollector sstableMetadataCollector = new MetadataCollector(cfs.metadata.comparator).replayPosition(context); - - return SSTableWriter.create(Descriptor.fromFilename(filename), (long) rows.size(), ActiveRepairService.UNREPAIRED_SSTABLE, cfs.metadata, cfs.partitioner, sstableMetadataCollector); + return SSTableWriter.create(Descriptor.fromFilename(filename), + (long)partitions.size(), + ActiveRepairService.UNREPAIRED_SSTABLE, + cfs.metadata, + cfs.partitioner, + sstableMetadataCollector, + new SerializationHeader(cfs.metadata, columns, stats)); } } @@ -427,17 +433,82 @@ public class Memtable implements Comparable { int rowOverhead; MemtableAllocator allocator = MEMORY_POOL.newAllocator(); - ConcurrentNavigableMap rows = new ConcurrentSkipListMap<>(); + ConcurrentNavigableMap partitions = new ConcurrentSkipListMap<>(); final Object val = new Object(); - for (int i = 0; i < count; i++) - rows.put(allocator.clone(new BufferDecoratedKey(new LongToken(i), ByteBufferUtil.EMPTY_BYTE_BUFFER), group), val); - double avgSize = ObjectSizes.measureDeep(rows) / (double) count; + for (int i = 0 ; i < count ; i++) + partitions.put(allocator.clone(new BufferDecoratedKey(new LongToken(i), ByteBufferUtil.EMPTY_BYTE_BUFFER), group), val); + double avgSize = ObjectSizes.measureDeep(partitions) / (double) count; rowOverhead = (int) ((avgSize - Math.floor(avgSize)) < 0.05 ? Math.floor(avgSize) : Math.ceil(avgSize)); rowOverhead -= ObjectSizes.measureDeep(new LongToken(0)); - rowOverhead += AtomicBTreeColumns.EMPTY_SIZE; + rowOverhead += AtomicBTreePartition.EMPTY_SIZE; allocator.setDiscarding(); allocator.setDiscarded(); return rowOverhead; } } + + private static class ColumnsCollector + { + private final HashMap predefined = new HashMap<>(); + private final ConcurrentSkipListSet extra = new ConcurrentSkipListSet<>(); + ColumnsCollector(PartitionColumns columns) + { + for (ColumnDefinition def : columns.statics) + predefined.put(def, new AtomicBoolean()); + for (ColumnDefinition def : columns.regulars) + predefined.put(def, new AtomicBoolean()); + } + + public void update(PartitionColumns columns) + { + for (ColumnDefinition s : columns.statics) + update(s); + for (ColumnDefinition r : columns.regulars) + update(r); + } + + private void update(ColumnDefinition definition) + { + AtomicBoolean present = predefined.get(definition); + if (present != null) + { + if (!present.get()) + present.set(true); + } + else + { + extra.add(definition); + } + } + + public PartitionColumns get() + { + PartitionColumns.Builder builder = PartitionColumns.builder(); + for (Map.Entry e : predefined.entrySet()) + if (e.getValue().get()) + builder.add(e.getKey()); + return builder.addAll(extra).build(); + } + } + + private static class StatsCollector + { + private final AtomicReference stats = new AtomicReference<>(RowStats.NO_STATS); + + public void update(RowStats newStats) + { + while (true) + { + RowStats current = stats.get(); + RowStats updated = current.mergeWith(newStats); + if (stats.compareAndSet(current, updated)) + return; + } + } + + public RowStats get() + { + return stats.get(); + } + } } diff --git a/src/java/org/apache/cassandra/db/MultiCBuilder.java b/src/java/org/apache/cassandra/db/MultiCBuilder.java new file mode 100644 index 0000000000..36a03baa5e --- /dev/null +++ b/src/java/org/apache/cassandra/db/MultiCBuilder.java @@ -0,0 +1,436 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.cassandra.db; + +import java.nio.ByteBuffer; +import java.util.*; + +import static java.util.Collections.singletonList; +import org.apache.cassandra.utils.ByteBufferUtil; +import org.apache.cassandra.utils.FBUtilities; + +/** + * Builder that allow to build multiple Clustering/Slice.Bound at the same time. + */ +public abstract class MultiCBuilder +{ + /** + * Creates a new empty {@code MultiCBuilder}. + */ + public static MultiCBuilder create(ClusteringComparator comparator) + { + return new ConcreteMultiCBuilder(comparator); + } + + /** + * Wraps an existing {@code CBuilder} to provide him with a MultiCBuilder interface + * for the sake of passing it to {@link Restriction.appendTo}. The resulting + * {@code MultiCBuilder} will still only be able to create a single clustering/bound + * and an {@code IllegalArgumentException} will be thrown if elements that added that + * would correspond to building multiple clusterings. + */ + public static MultiCBuilder wrap(final CBuilder builder) + { + return new MultiCBuilder() + { + private boolean containsNull; + private boolean containsUnset; + private boolean hasMissingElements; + + public MultiCBuilder addElementToAll(ByteBuffer value) + { + builder.add(value); + + if (value == null) + containsNull = true; + if (value == ByteBufferUtil.UNSET_BYTE_BUFFER) + containsUnset = true; + + return this; + } + + public MultiCBuilder addEachElementToAll(List values) + { + if (values.isEmpty()) + { + hasMissingElements = true; + return this; + } + + if (values.size() > 1) + throw new IllegalArgumentException(); + + return addElementToAll(values.get(0)); + } + + public MultiCBuilder addAllElementsToAll(List> values) + { + if (values.isEmpty()) + { + hasMissingElements = true; + return this; + } + + if (values.size() > 1) + throw new IllegalArgumentException(); + + return addEachElementToAll(values.get(0)); + } + + public int remainingCount() + { + return builder.remainingCount(); + } + + public boolean containsNull() + { + return containsNull; + } + + public boolean containsUnset() + { + return containsUnset; + } + + public boolean hasMissingElements() + { + return hasMissingElements; + } + + public NavigableSet build() + { + return FBUtilities.singleton(builder.build(), builder.comparator()); + } + + public SortedSet buildBound(boolean isStart, boolean isInclusive) + { + return FBUtilities.singleton(builder.buildBound(isStart, isInclusive), builder.comparator()); + } + }; + } + + /** + * Adds the specified element to all the clusterings. + *

+ * If this builder contains 2 clustering: A-B and A-C a call to this method to add D will result in the clusterings: + * A-B-D and A-C-D. + *

+ * + * @param value the value of the next element + * @return this MulitCBuilder + */ + public abstract MultiCBuilder addElementToAll(ByteBuffer value); + + /** + * Adds individually each of the specified elements to the end of all of the existing clusterings. + *

+ * If this builder contains 2 clusterings: A-B and A-C a call to this method to add D and E will result in the 4 + * clusterings: A-B-D, A-B-E, A-C-D and A-C-E. + *

+ * + * @param values the elements to add + * @return this CompositeBuilder + */ + public abstract MultiCBuilder addEachElementToAll(List values); + + /** + * Adds individually each of the specified list of elements to the end of all of the existing composites. + *

+ * If this builder contains 2 composites: A-B and A-C a call to this method to add [[D, E], [F, G]] will result in the 4 + * composites: A-B-D-E, A-B-F-G, A-C-D-E and A-C-F-G. + *

+ * + * @param values the elements to add + * @return this CompositeBuilder + */ + public abstract MultiCBuilder addAllElementsToAll(List> values); + + /** + * Returns the number of elements that can be added to the clusterings. + * + * @return the number of elements that can be added to the clusterings. + */ + public abstract int remainingCount(); + + /** + * Checks if the clusterings contains null elements. + * + * @return true if the clusterings contains null elements, false otherwise. + */ + public abstract boolean containsNull(); + + /** + * Checks if the clusterings contains unset elements. + * + * @return true if the clusterings contains unset elements, false otherwise. + */ + public abstract boolean containsUnset(); + + /** + * Checks if some empty list of values have been added + * @return true if the clusterings have some missing elements, false otherwise. + */ + public abstract boolean hasMissingElements(); + + /** + * Builds the clusterings. + * + * @return the clusterings + */ + public abstract NavigableSet build(); + + /** + * Builds the clusterings with the specified EOC. + * + * @return the clusterings + */ + public abstract SortedSet buildBound(boolean isStart, boolean isInclusive); + + /** + * Checks if some elements can still be added to the clusterings. + * + * @return true if it is possible to add more elements to the clusterings, false otherwise. + */ + public boolean hasRemaining() + { + return remainingCount() > 0; + } + + + private static class ConcreteMultiCBuilder extends MultiCBuilder + { + /** + * The table comparator. + */ + private final ClusteringComparator comparator; + + /** + * The elements of the clusterings + */ + private final List> elementsList = new ArrayList<>(); + + /** + * The number of elements that have been added. + */ + private int size; + + /** + * true if the clusterings have been build, false otherwise. + */ + private boolean built; + + /** + * true if the clusterings contains some null elements. + */ + private boolean containsNull; + + /** + * true if the composites contains some unset elements. + */ + private boolean containsUnset; + + /** + * true if some empty collection have been added. + */ + private boolean hasMissingElements; + + public ConcreteMultiCBuilder(ClusteringComparator comparator) + { + this.comparator = comparator; + } + + public MultiCBuilder addElementToAll(ByteBuffer value) + { + checkUpdateable(); + + if (isEmpty()) + elementsList.add(new ArrayList()); + + for (int i = 0, m = elementsList.size(); i < m; i++) + { + if (value == null) + containsNull = true; + if (value == ByteBufferUtil.UNSET_BYTE_BUFFER) + containsUnset = true; + + elementsList.get(i).add(value); + } + size++; + return this; + } + + public MultiCBuilder addEachElementToAll(List values) + { + checkUpdateable(); + + if (isEmpty()) + elementsList.add(new ArrayList()); + + if (values.isEmpty()) + { + hasMissingElements = true; + } + else + { + for (int i = 0, m = elementsList.size(); i < m; i++) + { + List oldComposite = elementsList.remove(0); + + for (int j = 0, n = values.size(); j < n; j++) + { + List newComposite = new ArrayList<>(oldComposite); + elementsList.add(newComposite); + + ByteBuffer value = values.get(j); + + if (value == null) + containsNull = true; + if (value == ByteBufferUtil.UNSET_BYTE_BUFFER) + containsUnset = true; + + newComposite.add(values.get(j)); + } + } + } + size++; + return this; + } + + public MultiCBuilder addAllElementsToAll(List> values) + { + checkUpdateable(); + + if (isEmpty()) + elementsList.add(new ArrayList()); + + if (values.isEmpty()) + { + hasMissingElements = true; + } + else + { + for (int i = 0, m = elementsList.size(); i < m; i++) + { + List oldComposite = elementsList.remove(0); + + for (int j = 0, n = values.size(); j < n; j++) + { + List newComposite = new ArrayList<>(oldComposite); + elementsList.add(newComposite); + + List value = values.get(j); + + if (value.isEmpty()) + hasMissingElements = true; + + if (value.contains(null)) + containsNull = true; + if (value.contains(ByteBufferUtil.UNSET_BYTE_BUFFER)) + containsUnset = true; + + newComposite.addAll(value); + } + } + size += values.get(0).size(); + } + return this; + } + + public int remainingCount() + { + return comparator.size() - size; + } + + /** + * Checks if this builder is empty. + * + * @return true if this builder is empty, false otherwise. + */ + public boolean isEmpty() + { + return elementsList.isEmpty(); + } + + public boolean containsNull() + { + return containsNull; + } + + public boolean containsUnset() + { + return containsUnset; + } + + public boolean hasMissingElements() + { + return hasMissingElements; + } + + public NavigableSet build() + { + built = true; + + if (hasMissingElements) + return FBUtilities.emptySortedSet(comparator); + + CBuilder builder = CBuilder.create(comparator); + + if (elementsList.isEmpty()) + return FBUtilities.singleton(builder.build(), builder.comparator()); + + // Use a TreeSet to sort and eliminate duplicates + NavigableSet set = new TreeSet<>(builder.comparator()); + + for (int i = 0, m = elementsList.size(); i < m; i++) + { + List elements = elementsList.get(i); + set.add(builder.buildWith(elements)); + } + return set; + } + + public SortedSet buildBound(boolean isStart, boolean isInclusive) + { + built = true; + + if (hasMissingElements) + return FBUtilities.emptySortedSet(comparator); + + CBuilder builder = CBuilder.create(comparator); + + if (elementsList.isEmpty()) + return FBUtilities.singleton(builder.buildBound(isStart, isInclusive), comparator); + + // Use a TreeSet to sort and eliminate duplicates + SortedSet set = new TreeSet<>(comparator); + + for (int i = 0, m = elementsList.size(); i < m; i++) + { + List elements = elementsList.get(i); + set.add(builder.buildBoundWith(elements, isStart, isInclusive)); + } + return set; + } + + private void checkUpdateable() + { + if (!hasRemaining() || built) + throw new IllegalStateException("this builder cannot be updated anymore"); + } + } +} diff --git a/src/java/org/apache/cassandra/db/Mutation.java b/src/java/org/apache/cassandra/db/Mutation.java index 9dd1686142..355d259917 100644 --- a/src/java/org/apache/cassandra/db/Mutation.java +++ b/src/java/org/apache/cassandra/db/Mutation.java @@ -19,7 +19,6 @@ package org.apache.cassandra.db; import java.io.DataInput; import java.io.IOException; -import java.nio.ByteBuffer; import java.util.*; import org.apache.commons.lang3.StringUtils; @@ -27,13 +26,15 @@ import org.apache.commons.lang3.StringUtils; import org.apache.cassandra.config.CFMetaData; import org.apache.cassandra.config.DatabaseDescriptor; import org.apache.cassandra.config.Schema; -import org.apache.cassandra.db.composites.CellName; -import org.apache.cassandra.db.composites.Composite; +import org.apache.cassandra.db.rows.SerializationHelper; +import org.apache.cassandra.db.partitions.*; import org.apache.cassandra.io.IVersionedSerializer; import org.apache.cassandra.io.util.DataOutputPlus; import org.apache.cassandra.net.MessageOut; import org.apache.cassandra.net.MessagingService; +import org.apache.cassandra.service.StorageService; import org.apache.cassandra.utils.ByteBufferUtil; + import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -51,37 +52,27 @@ public class Mutation implements IMutation // when we remove it, also restore SerializationsTest.testMutationRead to not regenerate new Mutations each test private final String keyspaceName; - private final ByteBuffer key; + private final DecoratedKey key; // map of column family id to mutations for that column family. - private final Map modifications; + private final Map modifications; - public Mutation(String keyspaceName, ByteBuffer key) + public Mutation(String keyspaceName, DecoratedKey key) { - this(keyspaceName, key, new HashMap()); + this(keyspaceName, key, new HashMap()); } - public Mutation(String keyspaceName, ByteBuffer key, ColumnFamily cf) + public Mutation(PartitionUpdate update) { - this(keyspaceName, key, Collections.singletonMap(cf.id(), cf)); + this(update.metadata().ksName, update.partitionKey(), Collections.singletonMap(update.metadata().cfId, update)); } - public Mutation(String keyspaceName, Row row) - { - this(keyspaceName, row.key.getKey(), row.cf); - } - - protected Mutation(String keyspaceName, ByteBuffer key, Map modifications) + protected Mutation(String keyspaceName, DecoratedKey key, Map modifications) { this.keyspaceName = keyspaceName; this.key = key; this.modifications = modifications; } - public Mutation(ByteBuffer key, ColumnFamily cf) - { - this(cf.metadata().ksName, key, cf); - } - public Mutation copy() { Mutation copy = new Mutation(keyspaceName, key, new HashMap<>(modifications)); @@ -98,53 +89,34 @@ public class Mutation implements IMutation return modifications.keySet(); } - public ByteBuffer key() + public DecoratedKey key() { return key; } - public Collection getColumnFamilies() + public Collection getPartitionUpdates() { return modifications.values(); } - public ColumnFamily getColumnFamily(UUID cfId) + public PartitionUpdate getPartitionUpdate(UUID cfId) { return modifications.get(cfId); } - /* - * Specify a column family name and the corresponding column - * family object. - * param @ cf - column family name - * param @ columnFamily - the column family. - */ - public void add(ColumnFamily columnFamily) + public Mutation add(PartitionUpdate update) { - assert columnFamily != null; - ColumnFamily prev = modifications.put(columnFamily.id(), columnFamily); + assert update != null; + PartitionUpdate prev = modifications.put(update.metadata().cfId, update); if (prev != null) // developer error - throw new IllegalArgumentException("Table " + columnFamily + " already has modifications in this mutation: " + prev); + throw new IllegalArgumentException("Table " + update.metadata().cfName + " already has modifications in this mutation: " + prev); + return this; } - /** - * @return the ColumnFamily in this Mutation corresponding to @param cfName, creating an empty one if necessary. - */ - public ColumnFamily addOrGet(String cfName) + public PartitionUpdate get(CFMetaData cfm) { - return addOrGet(Schema.instance.getCFMetaData(keyspaceName, cfName)); - } - - public ColumnFamily addOrGet(CFMetaData cfm) - { - ColumnFamily cf = modifications.get(cfm.cfId); - if (cf == null) - { - cf = ArrayBackedSortedColumns.factory.create(cfm); - modifications.put(cfm.cfId, cf); - } - return cf; + return modifications.get(cfm.cfId); } public boolean isEmpty() @@ -152,56 +124,56 @@ public class Mutation implements IMutation return modifications.isEmpty(); } - public void add(String cfName, CellName name, ByteBuffer value, long timestamp, int timeToLive) + /** + * Creates a new mutation that merges all the provided mutations. + * + * @param mutations the mutations to merge together. All mutation must be + * on the same keyspace and partition key. There should also be at least one + * mutation. + * @return a mutation that contains all the modifications contained in {@code mutations}. + * + * @throws IllegalArgumentException if not all the mutations are on the same + * keyspace and key. + */ + public static Mutation merge(List mutations) { - addOrGet(cfName).addColumn(name, value, timestamp, timeToLive); - } + assert !mutations.isEmpty(); - public void addCounter(String cfName, CellName name, long value) - { - addOrGet(cfName).addCounter(name, value); - } + if (mutations.size() == 1) + return mutations.get(0); - public void add(String cfName, CellName name, ByteBuffer value, long timestamp) - { - add(cfName, name, value, timestamp, 0); - } - - public void delete(String cfName, long timestamp) - { - int localDeleteTime = (int) (System.currentTimeMillis() / 1000); - addOrGet(cfName).delete(new DeletionInfo(timestamp, localDeleteTime)); - } - - public void delete(String cfName, CellName name, long timestamp) - { - int localDeleteTime = (int) (System.currentTimeMillis() / 1000); - addOrGet(cfName).addTombstone(name, localDeleteTime, timestamp); - } - - public void deleteRange(String cfName, Composite start, Composite end, long timestamp) - { - int localDeleteTime = (int) (System.currentTimeMillis() / 1000); - addOrGet(cfName).addAtom(new RangeTombstone(start, end, timestamp, localDeleteTime)); - } - - public void addAll(IMutation m) - { - if (!(m instanceof Mutation)) - throw new IllegalArgumentException(); - - Mutation mutation = (Mutation)m; - if (!keyspaceName.equals(mutation.keyspaceName) || !key.equals(mutation.key)) - throw new IllegalArgumentException(); - - for (Map.Entry entry : mutation.modifications.entrySet()) + Set updatedTables = new HashSet<>(); + String ks = null; + DecoratedKey key = null; + for (Mutation mutation : mutations) { - // It's slighty faster to assume the key wasn't present and fix if - // not in the case where it wasn't there indeed. - ColumnFamily cf = modifications.put(entry.getKey(), entry.getValue()); - if (cf != null) - entry.getValue().addAll(cf); + updatedTables.addAll(mutation.modifications.keySet()); + if (ks != null && !ks.equals(mutation.keyspaceName)) + throw new IllegalArgumentException(); + if (key != null && !key.equals(mutation.key)) + throw new IllegalArgumentException(); + ks = mutation.keyspaceName; + key = mutation.key; } + + List updates = new ArrayList<>(mutations.size()); + Map modifications = new HashMap<>(updatedTables.size()); + for (UUID table : updatedTables) + { + for (Mutation mutation : mutations) + { + PartitionUpdate upd = mutation.modifications.get(table); + if (upd != null) + updates.add(upd); + } + + if (updates.isEmpty()) + continue; + + modifications.put(table, updates.size() == 1 ? updates.get(0) : PartitionUpdate.merge(updates)); + updates.clear(); + } + return new Mutation(ks, key, modifications); } /* @@ -243,7 +215,7 @@ public class Mutation implements IMutation { StringBuilder buff = new StringBuilder("Mutation("); buff.append("keyspace='").append(keyspaceName).append('\''); - buff.append(", key='").append(ByteBufferUtil.bytesToHex(key)).append('\''); + buff.append(", key='").append(ByteBufferUtil.bytesToHex(key.getKey())).append('\''); buff.append(", modifications=["); if (shallow) { @@ -256,14 +228,16 @@ public class Mutation implements IMutation buff.append(StringUtils.join(cfnames, ", ")); } else - buff.append(StringUtils.join(modifications.values(), ", ")); + { + buff.append("\n ").append(StringUtils.join(modifications.values(), "\n ")).append("\n"); + } return buff.append("])").toString(); } public Mutation without(UUID cfId) { Mutation mutation = new Mutation(keyspaceName, key); - for (Map.Entry entry : modifications.entrySet()) + for (Map.Entry entry : modifications.entrySet()) if (!entry.getKey().equals(cfId)) mutation.add(entry.getValue()); return mutation; @@ -276,58 +250,52 @@ public class Mutation implements IMutation if (version < MessagingService.VERSION_20) out.writeUTF(mutation.getKeyspaceName()); - ByteBufferUtil.writeWithShortLength(mutation.key(), out); + if (version < MessagingService.VERSION_30) + ByteBufferUtil.writeWithShortLength(mutation.key().getKey(), out); /* serialize the modifications in the mutation */ int size = mutation.modifications.size(); out.writeInt(size); assert size > 0; - for (Map.Entry entry : mutation.modifications.entrySet()) - ColumnFamily.serializer.serialize(entry.getValue(), out, version); + for (Map.Entry entry : mutation.modifications.entrySet()) + PartitionUpdate.serializer.serialize(entry.getValue(), out, version); } - public Mutation deserialize(DataInput in, int version, ColumnSerializer.Flag flag) throws IOException + public Mutation deserialize(DataInput in, int version, SerializationHelper.Flag flag) throws IOException { String keyspaceName = null; // will always be set from cf.metadata but javac isn't smart enough to see that if (version < MessagingService.VERSION_20) keyspaceName = in.readUTF(); - ByteBuffer key = ByteBufferUtil.readWithShortLength(in); + DecoratedKey key = null; + if (version < MessagingService.VERSION_30) + key = StorageService.getPartitioner().decorateKey(ByteBufferUtil.readWithShortLength(in)); + int size = in.readInt(); assert size > 0; - Map modifications; if (size == 1) + return new Mutation(PartitionUpdate.serializer.deserialize(in, version, flag, key)); + + Map modifications = new HashMap<>(size); + PartitionUpdate update = null; + for (int i = 0; i < size; ++i) { - ColumnFamily cf = deserializeOneCf(in, version, flag); - modifications = Collections.singletonMap(cf.id(), cf); - keyspaceName = cf.metadata().ksName; - } - else - { - modifications = new HashMap(size); - for (int i = 0; i < size; ++i) - { - ColumnFamily cf = deserializeOneCf(in, version, flag); - modifications.put(cf.id(), cf); - keyspaceName = cf.metadata().ksName; - } + update = PartitionUpdate.serializer.deserialize(in, version, flag, key); + modifications.put(update.metadata().cfId, update); } + if (keyspaceName == null) + keyspaceName = update.metadata().ksName; + if (key == null) + key = update.partitionKey(); + return new Mutation(keyspaceName, key, modifications); } - private ColumnFamily deserializeOneCf(DataInput in, int version, ColumnSerializer.Flag flag) throws IOException - { - ColumnFamily cf = ColumnFamily.serializer.deserialize(in, ArrayBackedSortedColumns.factory, flag, version); - // We don't allow Mutation with null column family, so we should never get null back. - assert cf != null; - return cf; - } - public Mutation deserialize(DataInput in, int version) throws IOException { - return deserialize(in, version, ColumnSerializer.Flag.FROM_REMOTE); + return deserialize(in, version, SerializationHelper.Flag.FROM_REMOTE); } public long serializedSize(Mutation mutation, int version) @@ -338,12 +306,15 @@ public class Mutation implements IMutation if (version < MessagingService.VERSION_20) size += sizes.sizeof(mutation.getKeyspaceName()); - int keySize = mutation.key().remaining(); - size += sizes.sizeof((short) keySize) + keySize; + if (version < MessagingService.VERSION_30) + { + int keySize = mutation.key().getKey().remaining(); + size += sizes.sizeof((short) keySize) + keySize; + } size += sizes.sizeof(mutation.modifications.size()); - for (Map.Entry entry : mutation.modifications.entrySet()) - size += ColumnFamily.serializer.serializedSize(entry.getValue(), TypeSizes.NATIVE, version); + for (Map.Entry entry : mutation.modifications.entrySet()) + size += PartitionUpdate.serializer.serializedSize(entry.getValue(), version, sizes); return size; } diff --git a/src/java/org/apache/cassandra/db/NativeCell.java b/src/java/org/apache/cassandra/db/NativeCell.java deleted file mode 100644 index dac5674690..0000000000 --- a/src/java/org/apache/cassandra/db/NativeCell.java +++ /dev/null @@ -1,88 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.apache.cassandra.db; - -import java.security.MessageDigest; - -import org.apache.cassandra.config.CFMetaData; -import org.apache.cassandra.db.composites.CellName; -import org.apache.cassandra.utils.FBUtilities; -import org.apache.cassandra.utils.ObjectSizes; -import org.apache.cassandra.utils.concurrent.OpOrder; -import org.apache.cassandra.utils.memory.AbstractAllocator; -import org.apache.cassandra.utils.memory.MemtableAllocator; -import org.apache.cassandra.utils.memory.NativeAllocator; - -public class NativeCell extends AbstractNativeCell -{ - private static final long SIZE = ObjectSizes.measure(new NativeCell()); - - NativeCell() - {} - - public NativeCell(NativeAllocator allocator, OpOrder.Group writeOp, Cell copyOf) - { - super(allocator, writeOp, copyOf); - } - - @Override - public CellName name() - { - return this; - } - - @Override - public long timestamp() - { - return getLong(TIMESTAMP_OFFSET); - } - - @Override - public Cell localCopy(CFMetaData metadata, AbstractAllocator allocator) - { - return new BufferCell(copy(metadata, allocator), allocator.clone(value()), timestamp()); - } - - @Override - public Cell localCopy(CFMetaData metadata, MemtableAllocator allocator, OpOrder.Group opGroup) - { - return allocator.clone(this, metadata, opGroup); - } - - @Override - public void updateDigest(MessageDigest digest) - { - updateWithName(digest); // name - updateWithValue(digest); // value - - FBUtilities.updateWithLong(digest, timestamp()); - FBUtilities.updateWithByte(digest, serializationFlags()); - } - - @Override - public long unsharedHeapSizeExcludingData() - { - return SIZE; - } - - @Override - public long unsharedHeapSize() - { - return SIZE; - } -} diff --git a/src/java/org/apache/cassandra/db/NativeCounterCell.java b/src/java/org/apache/cassandra/db/NativeCounterCell.java deleted file mode 100644 index c16cc44fb7..0000000000 --- a/src/java/org/apache/cassandra/db/NativeCounterCell.java +++ /dev/null @@ -1,186 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.apache.cassandra.db; - -import java.security.MessageDigest; - -import org.apache.cassandra.config.CFMetaData; -import org.apache.cassandra.db.composites.CellNameType; -import org.apache.cassandra.serializers.MarshalException; -import org.apache.cassandra.utils.FBUtilities; -import org.apache.cassandra.utils.ObjectSizes; -import org.apache.cassandra.utils.concurrent.OpOrder; -import org.apache.cassandra.utils.memory.AbstractAllocator; -import org.apache.cassandra.utils.memory.MemtableAllocator; -import org.apache.cassandra.utils.memory.NativeAllocator; - -public class NativeCounterCell extends NativeCell implements CounterCell -{ - private static final long SIZE = ObjectSizes.measure(new NativeCounterCell()); - - private NativeCounterCell() - {} - - public NativeCounterCell(NativeAllocator allocator, OpOrder.Group writeOp, CounterCell copyOf) - { - super(allocator, writeOp, copyOf); - } - - @Override - protected void construct(Cell from) - { - super.construct(from); - setLong(internalSize() - 8, ((CounterCell) from).timestampOfLastDelete()); - } - - @Override - protected int postfixSize() - { - return 8; - } - - @Override - protected int sizeOf(Cell cell) - { - return 8 + super.sizeOf(cell); - } - - @Override - public long timestampOfLastDelete() - { - return getLong(internalSize() - 8); - } - - @Override - public long total() - { - return contextManager.total(value()); - } - - @Override - public boolean hasLegacyShards() - { - return contextManager.hasLegacyShards(value()); - } - - @Override - public Cell markLocalToBeCleared() - { - throw new UnsupportedOperationException(); - } - - @Override - public Cell diff(Cell cell) - { - return diffCounter(cell); - } - - @Override - public Cell reconcile(Cell cell) - { - return reconcileCounter(cell); - } - - @Override - public int serializationFlags() - { - return ColumnSerializer.COUNTER_MASK; - } - - @Override - public int cellDataSize() - { - // A counter column adds 8 bytes for timestampOfLastDelete to Cell. - return super.cellDataSize() + TypeSizes.NATIVE.sizeof(timestampOfLastDelete()); - } - - @Override - public int serializedSize(CellNameType type, TypeSizes typeSizes) - { - return super.serializedSize(type, typeSizes) + typeSizes.sizeof(timestampOfLastDelete()); - } - - @Override - public void validateFields(CFMetaData metadata) throws MarshalException - { - validateName(metadata); - // We cannot use the value validator as for other columns as the CounterColumnType validate a long, - // which is not the internal representation of counters - contextManager.validateContext(value()); - } - - /* - * We have to special case digest creation for counter column because - * we don't want to include the information about which shard of the - * context is a delta or not, since this information differs from node to - * node. - */ - @Override - public void updateDigest(MessageDigest digest) - { - updateWithName(digest); - - // We don't take the deltas into account in a digest - contextManager.updateDigest(digest, value()); - - FBUtilities.updateWithLong(digest, timestamp()); - FBUtilities.updateWithByte(digest, serializationFlags()); - FBUtilities.updateWithLong(digest, timestampOfLastDelete()); - } - - @Override - public String getString(CellNameType comparator) - { - return String.format("%s(%s:false:%s@%d!%d)", - getClass().getSimpleName(), - comparator.getString(name()), - contextManager.toString(value()), - timestamp(), - timestampOfLastDelete()); - } - - @Override - public CounterCell localCopy(CFMetaData metadata, AbstractAllocator allocator) - { - return new BufferCounterCell(copy(metadata, allocator), allocator.clone(value()), timestamp(), timestampOfLastDelete()); - } - - @Override - public CounterCell localCopy(CFMetaData metadata, MemtableAllocator allocator, OpOrder.Group opGroup) - { - return allocator.clone(this, metadata, opGroup); - } - - @Override - public long unsharedHeapSizeExcludingData() - { - return SIZE; - } - - @Override - public long unsharedHeapSize() - { - return SIZE; - } - - @Override - public boolean equals(Cell cell) - { - return super.equals(cell) && timestampOfLastDelete() == ((CounterCell) cell).timestampOfLastDelete(); - } -} diff --git a/src/java/org/apache/cassandra/db/NativeDeletedCell.java b/src/java/org/apache/cassandra/db/NativeDeletedCell.java deleted file mode 100644 index 6bdef43920..0000000000 --- a/src/java/org/apache/cassandra/db/NativeDeletedCell.java +++ /dev/null @@ -1,119 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.apache.cassandra.db; - -import java.security.MessageDigest; - -import org.apache.cassandra.config.CFMetaData; -import org.apache.cassandra.serializers.MarshalException; -import org.apache.cassandra.utils.FBUtilities; -import org.apache.cassandra.utils.ObjectSizes; -import org.apache.cassandra.utils.concurrent.OpOrder; -import org.apache.cassandra.utils.memory.AbstractAllocator; -import org.apache.cassandra.utils.memory.MemoryUtil; -import org.apache.cassandra.utils.memory.MemtableAllocator; -import org.apache.cassandra.utils.memory.NativeAllocator; - -public class NativeDeletedCell extends NativeCell implements DeletedCell -{ - private static final long SIZE = ObjectSizes.measure(new NativeDeletedCell()); - - private NativeDeletedCell() - {} - - public NativeDeletedCell(NativeAllocator allocator, OpOrder.Group writeOp, DeletedCell copyOf) - { - super(allocator, writeOp, copyOf); - } - - @Override - public Cell reconcile(Cell cell) - { - if (cell instanceof DeletedCell) - return super.reconcile(cell); - return cell.reconcile(this); - } - - @Override - public boolean isLive() - { - return false; - } - - @Override - public boolean isLive(long now) - { - return false; - } - - @Override - public int getLocalDeletionTime() - { - int v = getInt(valueStartOffset()); - return MemoryUtil.INVERTED_ORDER ? Integer.reverseBytes(v) : v; - } - - @Override - public int serializationFlags() - { - return ColumnSerializer.DELETION_MASK; - } - - @Override - public void validateFields(CFMetaData metadata) throws MarshalException - { - validateName(metadata); - - if ((int) (internalSize() - valueStartOffset()) != 4) - throw new MarshalException("A tombstone value should be 4 bytes long"); - if (getLocalDeletionTime() < 0) - throw new MarshalException("The local deletion time should not be negative"); - } - - @Override - public void updateDigest(MessageDigest digest) - { - updateWithName(digest); - FBUtilities.updateWithLong(digest, timestamp()); - FBUtilities.updateWithByte(digest, serializationFlags()); - } - - @Override - public DeletedCell localCopy(CFMetaData metadata, AbstractAllocator allocator) - { - return new BufferDeletedCell(copy(metadata, allocator), allocator.clone(value()), timestamp()); - } - - @Override - public DeletedCell localCopy(CFMetaData metadata, MemtableAllocator allocator, OpOrder.Group opGroup) - { - return allocator.clone(this, metadata, opGroup); - } - - @Override - public long unsharedHeapSizeExcludingData() - { - return SIZE; - } - - @Override - public long unsharedHeapSize() - { - return SIZE; - } -} diff --git a/src/java/org/apache/cassandra/db/NativeExpiringCell.java b/src/java/org/apache/cassandra/db/NativeExpiringCell.java deleted file mode 100644 index 6369536e09..0000000000 --- a/src/java/org/apache/cassandra/db/NativeExpiringCell.java +++ /dev/null @@ -1,190 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.apache.cassandra.db; - -import java.security.MessageDigest; - -import org.apache.cassandra.config.CFMetaData; -import org.apache.cassandra.db.composites.CellNameType; -import org.apache.cassandra.serializers.MarshalException; -import org.apache.cassandra.utils.FBUtilities; -import org.apache.cassandra.utils.ObjectSizes; -import org.apache.cassandra.utils.concurrent.OpOrder; -import org.apache.cassandra.utils.memory.AbstractAllocator; -import org.apache.cassandra.utils.memory.MemtableAllocator; -import org.apache.cassandra.utils.memory.NativeAllocator; - -public class NativeExpiringCell extends NativeCell implements ExpiringCell -{ - private static final long SIZE = ObjectSizes.measure(new NativeExpiringCell()); - - private NativeExpiringCell() - {} - - public NativeExpiringCell(NativeAllocator allocator, OpOrder.Group writeOp, ExpiringCell copyOf) - { - super(allocator, writeOp, copyOf); - } - - @Override - protected int sizeOf(Cell cell) - { - return super.sizeOf(cell) + 8; - } - - @Override - protected void construct(Cell from) - { - ExpiringCell expiring = (ExpiringCell) from; - - setInt(internalSize() - 4, expiring.getTimeToLive()); - setInt(internalSize() - 8, expiring.getLocalDeletionTime()); - super.construct(from); - } - - @Override - protected int postfixSize() - { - return 8; - } - - @Override - public int getTimeToLive() - { - return getInt(internalSize() - 4); - } - - @Override - public int getLocalDeletionTime() - { - return getInt(internalSize() - 8); - } - - @Override - public boolean isLive() - { - return isLive(System.currentTimeMillis()); - } - - @Override - public boolean isLive(long now) - { - return (int) (now / 1000) < getLocalDeletionTime(); - } - - @Override - public int serializationFlags() - { - return ColumnSerializer.EXPIRATION_MASK; - } - - @Override - public int cellDataSize() - { - return super.cellDataSize() + TypeSizes.NATIVE.sizeof(getLocalDeletionTime()) + TypeSizes.NATIVE.sizeof(getTimeToLive()); - } - - @Override - public int serializedSize(CellNameType type, TypeSizes typeSizes) - { - /* - * An expired column adds to a Cell : - * 4 bytes for the localExpirationTime - * + 4 bytes for the timeToLive - */ - return super.serializedSize(type, typeSizes) + typeSizes.sizeof(getLocalDeletionTime()) + typeSizes.sizeof(getTimeToLive()); - } - - @Override - public void validateFields(CFMetaData metadata) throws MarshalException - { - super.validateFields(metadata); - - if (getTimeToLive() <= 0) - throw new MarshalException("A column TTL should be > 0"); - if (getLocalDeletionTime() < 0) - throw new MarshalException("The local expiration time should not be negative"); - } - - @Override - public void updateDigest(MessageDigest digest) - { - super.updateDigest(digest); - FBUtilities.updateWithInt(digest, getTimeToLive()); - } - - @Override - public Cell reconcile(Cell cell) - { - long ts1 = timestamp(), ts2 = cell.timestamp(); - if (ts1 != ts2) - return ts1 < ts2 ? cell : this; - // we should prefer tombstones - if (cell instanceof DeletedCell) - return cell; - int c = value().compareTo(cell.value()); - if (c != 0) - return c < 0 ? cell : this; - // If we have same timestamp and value, prefer the longest ttl - if (cell instanceof ExpiringCell) - { - int let1 = getLocalDeletionTime(), let2 = cell.getLocalDeletionTime(); - if (let1 < let2) - return cell; - } - return this; - } - - public boolean equals(Cell cell) - { - if (!super.equals(cell)) - return false; - ExpiringCell that = (ExpiringCell) cell; - return getLocalDeletionTime() == that.getLocalDeletionTime() && getTimeToLive() == that.getTimeToLive(); - } - - @Override - public String getString(CellNameType comparator) - { - return String.format("%s(%s!%d)", getClass().getSimpleName(), super.getString(comparator), getTimeToLive()); - } - - @Override - public ExpiringCell localCopy(CFMetaData metadata, AbstractAllocator allocator) - { - return new BufferExpiringCell(name().copy(metadata, allocator), allocator.clone(value()), timestamp(), getTimeToLive(), getLocalDeletionTime()); - } - - @Override - public ExpiringCell localCopy(CFMetaData metadata, MemtableAllocator allocator, OpOrder.Group opGroup) - { - return allocator.clone(this, metadata, opGroup); - } - - @Override - public long unsharedHeapSizeExcludingData() - { - return SIZE; - } - - @Override - public long unsharedHeapSize() - { - return SIZE; - } -} diff --git a/src/java/org/apache/cassandra/db/OnDiskAtom.java b/src/java/org/apache/cassandra/db/OnDiskAtom.java deleted file mode 100644 index f5eddb9476..0000000000 --- a/src/java/org/apache/cassandra/db/OnDiskAtom.java +++ /dev/null @@ -1,102 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.apache.cassandra.db; - -import java.io.*; -import java.security.MessageDigest; - -import org.apache.cassandra.config.CFMetaData; -import org.apache.cassandra.db.composites.CellName; -import org.apache.cassandra.db.composites.CellNameType; -import org.apache.cassandra.db.composites.Composite; -import org.apache.cassandra.io.ISSTableSerializer; -import org.apache.cassandra.io.sstable.format.Version; -import org.apache.cassandra.io.util.DataOutputPlus; -import org.apache.cassandra.serializers.MarshalException; - -public interface OnDiskAtom -{ - public Composite name(); - - /** - * For a standard column, this is the same as timestamp(). - * For a super column, this is the min/max column timestamp of the sub columns. - */ - public long timestamp(); - public int getLocalDeletionTime(); // for tombstone GC, so int is sufficient granularity - - public void validateFields(CFMetaData metadata) throws MarshalException; - public void updateDigest(MessageDigest digest); - - public static class Serializer implements ISSTableSerializer - { - private final CellNameType type; - - public Serializer(CellNameType type) - { - this.type = type; - } - - public void serializeForSSTable(OnDiskAtom atom, DataOutputPlus out) throws IOException - { - if (atom instanceof Cell) - { - type.columnSerializer().serialize((Cell)atom, out); - } - else - { - assert atom instanceof RangeTombstone; - type.rangeTombstoneSerializer().serializeForSSTable((RangeTombstone)atom, out); - } - } - - public OnDiskAtom deserializeFromSSTable(DataInput in, Version version) throws IOException - { - return deserializeFromSSTable(in, ColumnSerializer.Flag.LOCAL, Integer.MIN_VALUE, version); - } - - public OnDiskAtom deserializeFromSSTable(DataInput in, ColumnSerializer.Flag flag, int expireBefore, Version version) throws IOException - { - Composite name = type.serializer().deserialize(in); - if (name.isEmpty()) - { - // SSTableWriter.END_OF_ROW - return null; - } - - int b = in.readUnsignedByte(); - if ((b & ColumnSerializer.RANGE_TOMBSTONE_MASK) != 0) - return type.rangeTombstoneSerializer().deserializeBody(in, name, version); - else - return type.columnSerializer().deserializeColumnBody(in, (CellName)name, b, flag, expireBefore); - } - - public long serializedSizeForSSTable(OnDiskAtom atom) - { - if (atom instanceof Cell) - { - return type.columnSerializer().serializedSize((Cell)atom, TypeSizes.NATIVE); - } - else - { - assert atom instanceof RangeTombstone; - return type.rangeTombstoneSerializer().serializedSizeForSSTable((RangeTombstone)atom); - } - } - } -} diff --git a/src/java/org/apache/cassandra/db/PagedRangeCommand.java b/src/java/org/apache/cassandra/db/PagedRangeCommand.java deleted file mode 100644 index 40ef88ef6b..0000000000 --- a/src/java/org/apache/cassandra/db/PagedRangeCommand.java +++ /dev/null @@ -1,224 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.apache.cassandra.db; - -import java.io.DataInput; -import java.io.IOException; -import java.util.ArrayList; -import java.util.List; - -import org.apache.cassandra.config.CFMetaData; -import org.apache.cassandra.config.Schema; -import org.apache.cassandra.db.composites.Composite; -import org.apache.cassandra.db.filter.*; -import org.apache.cassandra.dht.AbstractBounds; -import org.apache.cassandra.io.IVersionedSerializer; -import org.apache.cassandra.io.util.DataOutputPlus; -import org.apache.cassandra.net.MessageOut; -import org.apache.cassandra.net.MessagingService; - -public class PagedRangeCommand extends AbstractRangeCommand -{ - public static final IVersionedSerializer serializer = new Serializer(); - - public final Composite start; - public final Composite stop; - public final int limit; - private final boolean countCQL3Rows; - - public PagedRangeCommand(String keyspace, - String columnFamily, - long timestamp, - AbstractBounds keyRange, - SliceQueryFilter predicate, - Composite start, - Composite stop, - List rowFilter, - int limit, - boolean countCQL3Rows) - { - super(keyspace, columnFamily, timestamp, keyRange, predicate, rowFilter); - this.start = start; - this.stop = stop; - this.limit = limit; - this.countCQL3Rows = countCQL3Rows; - } - - public MessageOut createMessage() - { - return new MessageOut<>(MessagingService.Verb.PAGED_RANGE, this, serializer); - } - - public AbstractRangeCommand forSubRange(AbstractBounds subRange) - { - Composite newStart = subRange.left.equals(keyRange.left) ? start : ((SliceQueryFilter)predicate).start(); - Composite newStop = subRange.right.equals(keyRange.right) ? stop : ((SliceQueryFilter)predicate).finish(); - return new PagedRangeCommand(keyspace, - columnFamily, - timestamp, - subRange, - ((SliceQueryFilter) predicate).cloneShallow(), - newStart, - newStop, - rowFilter, - limit, - countCQL3Rows); - } - - public AbstractRangeCommand withUpdatedLimit(int newLimit) - { - return new PagedRangeCommand(keyspace, - columnFamily, - timestamp, - keyRange, - ((SliceQueryFilter) predicate).cloneShallow(), - start, - stop, - rowFilter, - newLimit, - countCQL3Rows); - } - - public int limit() - { - return limit; - } - - public boolean countCQL3Rows() - { - return countCQL3Rows; - } - - public List executeLocally() - { - ColumnFamilyStore cfs = Keyspace.open(keyspace).getColumnFamilyStore(columnFamily); - - ExtendedFilter exFilter = cfs.makeExtendedFilter(keyRange, (SliceQueryFilter)predicate, start, stop, rowFilter, limit, countCQL3Rows(), timestamp); - if (cfs.indexManager.hasIndexFor(rowFilter)) - return cfs.search(exFilter); - else - return cfs.getRangeSlice(exFilter); - } - - @Override - public String toString() - { - return String.format("PagedRange(%s, %s, %d, %s, %s, %s, %s, %s, %d)", keyspace, columnFamily, timestamp, keyRange, predicate, start, stop, rowFilter, limit); - } - - private static class Serializer implements IVersionedSerializer - { - public void serialize(PagedRangeCommand cmd, DataOutputPlus out, int version) throws IOException - { - out.writeUTF(cmd.keyspace); - out.writeUTF(cmd.columnFamily); - out.writeLong(cmd.timestamp); - - MessagingService.validatePartitioner(cmd.keyRange); - AbstractBounds.rowPositionSerializer.serialize(cmd.keyRange, out, version); - - CFMetaData metadata = Schema.instance.getCFMetaData(cmd.keyspace, cmd.columnFamily); - - // SliceQueryFilter (the count is not used) - SliceQueryFilter filter = (SliceQueryFilter)cmd.predicate; - metadata.comparator.sliceQueryFilterSerializer().serialize(filter, out, version); - - // The start and stop of the page - metadata.comparator.serializer().serialize(cmd.start, out); - metadata.comparator.serializer().serialize(cmd.stop, out); - - out.writeInt(cmd.rowFilter.size()); - for (IndexExpression expr : cmd.rowFilter) - { - expr.writeTo(out);; - } - - out.writeInt(cmd.limit); - if (version >= MessagingService.VERSION_21) - out.writeBoolean(cmd.countCQL3Rows); - } - - public PagedRangeCommand deserialize(DataInput in, int version) throws IOException - { - String keyspace = in.readUTF(); - String columnFamily = in.readUTF(); - long timestamp = in.readLong(); - - AbstractBounds keyRange = - AbstractBounds.rowPositionSerializer.deserialize(in, MessagingService.globalPartitioner(), version); - - CFMetaData metadata = Schema.instance.getCFMetaData(keyspace, columnFamily); - if (metadata == null) - { - String message = String.format("Got paged range command for nonexistent table %s.%s. If the table was just " + - "created, this is likely due to the schema not being fully propagated. Please wait for schema " + - "agreement on table creation." , keyspace, columnFamily); - throw new UnknownColumnFamilyException(message, null); - } - - SliceQueryFilter predicate = metadata.comparator.sliceQueryFilterSerializer().deserialize(in, version); - - Composite start = metadata.comparator.serializer().deserialize(in); - Composite stop = metadata.comparator.serializer().deserialize(in); - - int filterCount = in.readInt(); - List rowFilter = new ArrayList(filterCount); - for (int i = 0; i < filterCount; i++) - { - rowFilter.add(IndexExpression.readFrom(in)); - } - - int limit = in.readInt(); - boolean countCQL3Rows = version >= MessagingService.VERSION_21 - ? in.readBoolean() - : predicate.compositesToGroup >= 0 || predicate.count != 1; // See #6857 - return new PagedRangeCommand(keyspace, columnFamily, timestamp, keyRange, predicate, start, stop, rowFilter, limit, countCQL3Rows); - } - - public long serializedSize(PagedRangeCommand cmd, int version) - { - long size = 0; - - size += TypeSizes.NATIVE.sizeof(cmd.keyspace); - size += TypeSizes.NATIVE.sizeof(cmd.columnFamily); - size += TypeSizes.NATIVE.sizeof(cmd.timestamp); - - size += AbstractBounds.rowPositionSerializer.serializedSize(cmd.keyRange, version); - - CFMetaData metadata = Schema.instance.getCFMetaData(cmd.keyspace, cmd.columnFamily); - - size += metadata.comparator.sliceQueryFilterSerializer().serializedSize((SliceQueryFilter)cmd.predicate, version); - - size += metadata.comparator.serializer().serializedSize(cmd.start, TypeSizes.NATIVE); - size += metadata.comparator.serializer().serializedSize(cmd.stop, TypeSizes.NATIVE); - - size += TypeSizes.NATIVE.sizeof(cmd.rowFilter.size()); - for (IndexExpression expr : cmd.rowFilter) - { - size += TypeSizes.NATIVE.sizeofWithShortLength(expr.column); - size += TypeSizes.NATIVE.sizeof(expr.operator.ordinal()); - size += TypeSizes.NATIVE.sizeofWithShortLength(expr.value); - } - - size += TypeSizes.NATIVE.sizeof(cmd.limit); - if (version >= MessagingService.VERSION_21) - size += TypeSizes.NATIVE.sizeof(cmd.countCQL3Rows); - return size; - } - } -} diff --git a/src/java/org/apache/cassandra/db/PartitionColumns.java b/src/java/org/apache/cassandra/db/PartitionColumns.java new file mode 100644 index 0000000000..a1b1d0086e --- /dev/null +++ b/src/java/org/apache/cassandra/db/PartitionColumns.java @@ -0,0 +1,184 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.cassandra.db; + +import java.util.*; +import java.security.MessageDigest; + +import com.google.common.collect.Iterators; + +import org.apache.cassandra.config.ColumnDefinition; + +/** + * Columns (or a subset of the columns) that a partition contains. + * This mainly groups both static and regular columns for convenience. + */ +public class PartitionColumns implements Iterable +{ + public static PartitionColumns NONE = new PartitionColumns(Columns.NONE, Columns.NONE); + + public final Columns statics; + public final Columns regulars; + + public PartitionColumns(Columns statics, Columns regulars) + { + this.statics = statics; + this.regulars = regulars; + } + + public static PartitionColumns of(ColumnDefinition column) + { + return new PartitionColumns(column.isStatic() ? Columns.of(column) : Columns.NONE, + column.isStatic() ? Columns.NONE : Columns.of(column)); + } + + public PartitionColumns without(ColumnDefinition column) + { + return new PartitionColumns(column.isStatic() ? statics.without(column) : statics, + column.isStatic() ? regulars : regulars.without(column)); + } + + public PartitionColumns withoutStatics() + { + return statics.isEmpty() ? this : new PartitionColumns(Columns.NONE, regulars); + } + + public boolean isEmpty() + { + return statics.isEmpty() && regulars.isEmpty(); + } + + public boolean contains(ColumnDefinition column) + { + return column.isStatic() ? statics.contains(column) : regulars.contains(column); + } + + public boolean includes(PartitionColumns columns) + { + return statics.contains(columns.statics) && regulars.contains(columns.regulars); + } + + public Iterator iterator() + { + return Iterators.concat(statics.iterator(), regulars.iterator()); + } + + public Iterator selectOrderIterator() + { + return Iterators.concat(statics.selectOrderIterator(), regulars.selectOrderIterator()); + } + + @Override + public String toString() + { + StringBuilder sb = new StringBuilder(); + sb.append("[").append(statics).append(" | ").append(regulars).append("]"); + return sb.toString(); + } + + @Override + public boolean equals(Object other) + { + if (!(other instanceof PartitionColumns)) + return false; + + PartitionColumns that = (PartitionColumns)other; + return this.statics.equals(that.statics) + && this.regulars.equals(that.regulars); + } + + @Override + public int hashCode() + { + return Objects.hash(statics, regulars); + } + + public void digest(MessageDigest digest) + { + regulars.digest(digest); + statics.digest(digest); + } + + public static Builder builder() + { + return new Builder(); + } + + public static class Builder + { + // Note that we do want to use sorted sets because we want the column definitions to be compared + // through compareTo, not equals. The former basically check it's the same column name, while the latter + // check it's the same object, including the same type. + private SortedSet regularColumns; + private SortedSet staticColumns; + + public Builder add(ColumnDefinition c) + { + if (c.isStatic()) + { + if (staticColumns == null) + staticColumns = new TreeSet<>(); + staticColumns.add(c); + } + else + { + assert c.isRegular(); + if (regularColumns == null) + regularColumns = new TreeSet<>(); + regularColumns.add(c); + } + return this; + } + + public int added() + { + return (regularColumns == null ? 0 : regularColumns.size()) + + (staticColumns == null ? 0 : staticColumns.size()); + } + + public Builder addAll(Iterable columns) + { + for (ColumnDefinition c : columns) + add(c); + return this; + } + + public Builder addAll(PartitionColumns columns) + { + if (regularColumns == null && !columns.regulars.isEmpty()) + regularColumns = new TreeSet<>(); + + for (ColumnDefinition c : columns.regulars) + regularColumns.add(c); + + if (staticColumns == null && !columns.statics.isEmpty()) + staticColumns = new TreeSet<>(); + + for (ColumnDefinition c : columns.statics) + staticColumns.add(c); + + return this; + } + + public PartitionColumns build() + { + return new PartitionColumns(staticColumns == null ? Columns.NONE : Columns.from(staticColumns), + regularColumns == null ? Columns.NONE : Columns.from(regularColumns)); + } + } +} diff --git a/src/java/org/apache/cassandra/db/RowPosition.java b/src/java/org/apache/cassandra/db/PartitionPosition.java similarity index 88% rename from src/java/org/apache/cassandra/db/RowPosition.java rename to src/java/org/apache/cassandra/db/PartitionPosition.java index 3fa0465975..1dc940e432 100644 --- a/src/java/org/apache/cassandra/db/RowPosition.java +++ b/src/java/org/apache/cassandra/db/PartitionPosition.java @@ -22,12 +22,11 @@ import java.io.IOException; import java.nio.ByteBuffer; import org.apache.cassandra.dht.*; -import org.apache.cassandra.io.ISerializer; import org.apache.cassandra.io.util.DataOutputPlus; import org.apache.cassandra.service.StorageService; import org.apache.cassandra.utils.ByteBufferUtil; -public interface RowPosition extends RingPosition +public interface PartitionPosition extends RingPosition { public static enum Kind { @@ -45,7 +44,7 @@ public interface RowPosition extends RingPosition public static final class ForKey { - public static RowPosition get(ByteBuffer key, IPartitioner p) + public static PartitionPosition get(ByteBuffer key, IPartitioner p) { return key == null || key.remaining() == 0 ? p.getMinimumToken().minKeyBound() : p.decorateKey(key); } @@ -56,7 +55,7 @@ public interface RowPosition extends RingPosition public Kind kind(); public boolean isMinimum(); - public static class RowPositionSerializer implements IPartitionerDependentSerializer + public static class RowPositionSerializer implements IPartitionerDependentSerializer { /* * We need to be able to serialize both Token.KeyBound and @@ -69,7 +68,7 @@ public interface RowPosition extends RingPosition * token is recreated on the other side). In the other cases, we then * serialize the token. */ - public void serialize(RowPosition pos, DataOutputPlus out, int version) throws IOException + public void serialize(PartitionPosition pos, DataOutputPlus out, int version) throws IOException { Kind kind = pos.kind(); out.writeByte(kind.ordinal()); @@ -79,7 +78,7 @@ public interface RowPosition extends RingPosition Token.serializer.serialize(pos.getToken(), out, version); } - public RowPosition deserialize(DataInput in, IPartitioner p, int version) throws IOException + public PartitionPosition deserialize(DataInput in, IPartitioner p, int version) throws IOException { Kind kind = Kind.fromOrdinal(in.readByte()); if (kind == Kind.ROW_KEY) @@ -94,7 +93,7 @@ public interface RowPosition extends RingPosition } } - public long serializedSize(RowPosition pos, int version) + public long serializedSize(PartitionPosition pos, int version) { Kind kind = pos.kind(); int size = 1; // 1 byte for enum diff --git a/src/java/org/apache/cassandra/db/PartitionRangeReadCommand.java b/src/java/org/apache/cassandra/db/PartitionRangeReadCommand.java new file mode 100644 index 0000000000..c11a9bed07 --- /dev/null +++ b/src/java/org/apache/cassandra/db/PartitionRangeReadCommand.java @@ -0,0 +1,288 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.cassandra.db; + +import java.io.DataInput; +import java.io.IOException; +import java.util.ArrayList; +import java.util.List; + +import com.google.common.collect.Iterables; + +import org.apache.cassandra.config.CFMetaData; +import org.apache.cassandra.config.DatabaseDescriptor; +import org.apache.cassandra.db.rows.*; +import org.apache.cassandra.db.filter.*; +import org.apache.cassandra.db.partitions.*; +import org.apache.cassandra.db.index.SecondaryIndexSearcher; +import org.apache.cassandra.dht.AbstractBounds; +import org.apache.cassandra.exceptions.RequestExecutionException; +import org.apache.cassandra.io.util.DataOutputPlus; +import org.apache.cassandra.io.sstable.format.SSTableReader; +import org.apache.cassandra.metrics.ColumnFamilyMetrics; +import org.apache.cassandra.service.*; +import org.apache.cassandra.service.pager.*; +import org.apache.cassandra.thrift.ThriftResultsMerger; +import org.apache.cassandra.tracing.Tracing; +import org.apache.cassandra.utils.FBUtilities; + +/** + * A read command that selects a (part of a) range of partitions. + */ +public class PartitionRangeReadCommand extends ReadCommand +{ + protected static final SelectionDeserializer selectionDeserializer = new Deserializer(); + + private final DataRange dataRange; + + public PartitionRangeReadCommand(boolean isDigest, + boolean isForThrift, + CFMetaData metadata, + int nowInSec, + ColumnFilter columnFilter, + RowFilter rowFilter, + DataLimits limits, + DataRange dataRange) + { + super(Kind.PARTITION_RANGE, isDigest, isForThrift, metadata, nowInSec, columnFilter, rowFilter, limits); + this.dataRange = dataRange; + } + + public PartitionRangeReadCommand(CFMetaData metadata, + int nowInSec, + ColumnFilter columnFilter, + RowFilter rowFilter, + DataLimits limits, + DataRange dataRange) + { + this(false, false, metadata, nowInSec, columnFilter, rowFilter, limits, dataRange); + } + + /** + * Creates a new read command that query all the data in the table. + * + * @param metadata the table to query. + * @param nowInSec the time in seconds to use are "now" for this query. + * + * @return a newly created read command that queries everything in the table. + */ + public static PartitionRangeReadCommand allDataRead(CFMetaData metadata, int nowInSec) + { + return new PartitionRangeReadCommand(metadata, + nowInSec, + ColumnFilter.all(metadata), + RowFilter.NONE, + DataLimits.NONE, + DataRange.allData(StorageService.getPartitioner())); + } + + public DataRange dataRange() + { + return dataRange; + } + + public ClusteringIndexFilter clusteringIndexFilter(DecoratedKey key) + { + return dataRange.clusteringIndexFilter(key); + } + + public boolean isNamesQuery() + { + return dataRange.isNamesQuery(); + } + + public PartitionRangeReadCommand forSubRange(AbstractBounds range) + { + return new PartitionRangeReadCommand(isDigestQuery(), isForThrift(), metadata(), nowInSec(), columnFilter(), rowFilter(), limits(), dataRange().forSubRange(range)); + } + + public PartitionRangeReadCommand copy() + { + return new PartitionRangeReadCommand(isDigestQuery(), isForThrift(), metadata(), nowInSec(), columnFilter(), rowFilter(), limits(), dataRange()); + } + + public PartitionRangeReadCommand withUpdatedLimit(DataLimits newLimits) + { + return new PartitionRangeReadCommand(metadata(), nowInSec(), columnFilter(), rowFilter(), newLimits, dataRange()); + } + + public long getTimeout() + { + return DatabaseDescriptor.getRangeRpcTimeout(); + } + + public boolean selects(DecoratedKey partitionKey, Clustering clustering) + { + if (!dataRange().contains(partitionKey)) + return false; + + if (clustering == Clustering.STATIC_CLUSTERING) + return !columnFilter().fetchedColumns().statics.isEmpty(); + + return dataRange().clusteringIndexFilter(partitionKey).selects(clustering); + } + + public PartitionIterator execute(ConsistencyLevel consistency, ClientState clientState) throws RequestExecutionException + { + return StorageProxy.getRangeSlice(this, consistency); + } + + public QueryPager getPager(PagingState pagingState) + { + if (isNamesQuery()) + return new RangeNamesQueryPager(this, pagingState); + else + return new RangeSliceQueryPager(this, pagingState); + } + + protected void recordLatency(ColumnFamilyMetrics metric, long latencyNanos) + { + metric.rangeLatency.addNano(latencyNanos); + } + + protected UnfilteredPartitionIterator queryStorage(final ColumnFamilyStore cfs, ReadOrderGroup orderGroup) + { + ColumnFamilyStore.ViewFragment view = cfs.select(cfs.viewFilter(dataRange().keyRange())); + Tracing.trace("Executing seq scan across {} sstables for {}", view.sstables.size(), dataRange().keyRange().getString(metadata().getKeyValidator())); + + // fetch data from current memtable, historical memtables, and SSTables in the correct order. + final List iterators = new ArrayList<>(Iterables.size(view.memtables) + view.sstables.size()); + + try + { + for (Memtable memtable : view.memtables) + { + @SuppressWarnings("resource") // We close on exception and on closing the result returned by this method + UnfilteredPartitionIterator iter = memtable.makePartitionIterator(columnFilter(), dataRange(), isForThrift()); + iterators.add(isForThrift() ? ThriftResultsMerger.maybeWrap(iter, metadata(), nowInSec()) : iter); + } + + for (SSTableReader sstable : view.sstables) + { + @SuppressWarnings("resource") // We close on exception and on closing the result returned by this method + UnfilteredPartitionIterator iter = sstable.getScanner(columnFilter(), dataRange(), isForThrift()); + iterators.add(isForThrift() ? ThriftResultsMerger.maybeWrap(iter, metadata(), nowInSec()) : iter); + } + + return checkCacheFilter(UnfilteredPartitionIterators.mergeLazily(iterators, nowInSec()), cfs); + } + catch (RuntimeException | Error e) + { + try + { + FBUtilities.closeAll(iterators); + } + catch (Exception suppressed) + { + e.addSuppressed(suppressed); + } + + throw e; + } + } + + private UnfilteredPartitionIterator checkCacheFilter(UnfilteredPartitionIterator iter, final ColumnFamilyStore cfs) + { + return new WrappingUnfilteredPartitionIterator(iter) + { + @Override + public UnfilteredRowIterator computeNext(UnfilteredRowIterator iter) + { + // Note that we rely on the fact that until we actually advance 'iter', no really costly operation is actually done + // (except for reading the partition key from the index file) due to the call to mergeLazily in queryStorage. + DecoratedKey dk = iter.partitionKey(); + + // Check if this partition is in the rowCache and if it is, if it covers our filter + CachedPartition cached = cfs.getRawCachedPartition(dk); + ClusteringIndexFilter filter = dataRange().clusteringIndexFilter(dk); + + if (cached != null && cfs.isFilterFullyCoveredBy(filter, limits(), cached, nowInSec())) + { + // We won't use 'iter' so close it now. + iter.close(); + + return filter.getUnfilteredRowIterator(columnFilter(), cached); + } + + return iter; + } + }; + } + + protected void appendCQLWhereClause(StringBuilder sb) + { + if (dataRange.isUnrestricted() && rowFilter().isEmpty()) + return; + + sb.append(" WHERE "); + // We put the row filter first because the data range can end by "ORDER BY" + if (!rowFilter().isEmpty()) + { + sb.append(rowFilter()); + if (!dataRange.isUnrestricted()) + sb.append(" AND "); + } + if (!dataRange.isUnrestricted()) + sb.append(dataRange.toCQLString(metadata())); + } + + /** + * Allow to post-process the result of the query after it has been reconciled on the coordinator + * but before it is passed to the CQL layer to return the ResultSet. + * + * See CASSANDRA-8717 for why this exists. + */ + public PartitionIterator postReconciliationProcessing(PartitionIterator result) + { + ColumnFamilyStore cfs = Keyspace.open(metadata().ksName).getColumnFamilyStore(metadata().cfName); + SecondaryIndexSearcher searcher = getIndexSearcher(cfs); + return searcher == null ? result : searcher.postReconciliationProcessing(rowFilter(), result); + } + + @Override + public String toString() + { + return String.format("Read(%s.%s columns=%s rowfilter=%s limits=%s %s)", + metadata().ksName, + metadata().cfName, + columnFilter(), + rowFilter(), + limits(), + dataRange().toString(metadata())); + } + + protected void serializeSelection(DataOutputPlus out, int version) throws IOException + { + DataRange.serializer.serialize(dataRange(), out, version, metadata()); + } + + protected long selectionSerializedSize(int version) + { + return DataRange.serializer.serializedSize(dataRange(), version, metadata()); + } + + private static class Deserializer extends SelectionDeserializer + { + public ReadCommand deserialize(DataInput in, int version, boolean isDigest, boolean isForThrift, CFMetaData metadata, int nowInSec, ColumnFilter columnFilter, RowFilter rowFilter, DataLimits limits) + throws IOException + { + DataRange range = DataRange.serializer.deserialize(in, version, metadata); + return new PartitionRangeReadCommand(isDigest, isForThrift, metadata, nowInSec, columnFilter, rowFilter, limits, range); + } + }; +} diff --git a/src/java/org/apache/cassandra/db/RangeSliceCommand.java b/src/java/org/apache/cassandra/db/RangeSliceCommand.java deleted file mode 100644 index 664eeee35e..0000000000 --- a/src/java/org/apache/cassandra/db/RangeSliceCommand.java +++ /dev/null @@ -1,246 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.apache.cassandra.db; - -import java.io.DataInput; -import java.io.IOException; -import java.util.ArrayList; -import java.util.List; - -import com.google.common.base.Objects; - -import org.apache.cassandra.config.CFMetaData; -import org.apache.cassandra.config.Schema; -import org.apache.cassandra.db.filter.ExtendedFilter; -import org.apache.cassandra.db.filter.IDiskAtomFilter; -import org.apache.cassandra.dht.AbstractBounds; -import org.apache.cassandra.io.IVersionedSerializer; -import org.apache.cassandra.io.util.DataOutputPlus; -import org.apache.cassandra.net.MessageOut; -import org.apache.cassandra.net.MessagingService; -import org.apache.cassandra.service.pager.Pageable; - -public class RangeSliceCommand extends AbstractRangeCommand implements Pageable -{ - public static final RangeSliceCommandSerializer serializer = new RangeSliceCommandSerializer(); - - public final int maxResults; - public final boolean countCQL3Rows; - public final boolean isPaging; - - public RangeSliceCommand(String keyspace, - String columnFamily, - long timestamp, - IDiskAtomFilter predicate, - AbstractBounds range, - int maxResults) - { - this(keyspace, columnFamily, timestamp, predicate, range, null, maxResults, false, false); - } - - public RangeSliceCommand(String keyspace, - String columnFamily, - long timestamp, - IDiskAtomFilter predicate, - AbstractBounds range, - List row_filter, - int maxResults) - { - this(keyspace, columnFamily, timestamp, predicate, range, row_filter, maxResults, false, false); - } - - public RangeSliceCommand(String keyspace, - String columnFamily, - long timestamp, - IDiskAtomFilter predicate, - AbstractBounds range, - List rowFilter, - int maxResults, - boolean countCQL3Rows, - boolean isPaging) - { - super(keyspace, columnFamily, timestamp, range, predicate, rowFilter); - this.maxResults = maxResults; - this.countCQL3Rows = countCQL3Rows; - this.isPaging = isPaging; - } - - public MessageOut createMessage() - { - return new MessageOut<>(MessagingService.Verb.RANGE_SLICE, this, serializer); - } - - public AbstractRangeCommand forSubRange(AbstractBounds subRange) - { - return new RangeSliceCommand(keyspace, - columnFamily, - timestamp, - predicate.cloneShallow(), - subRange, - rowFilter, - maxResults, - countCQL3Rows, - isPaging); - } - - public AbstractRangeCommand withUpdatedLimit(int newLimit) - { - return new RangeSliceCommand(keyspace, - columnFamily, - timestamp, - predicate.cloneShallow(), - keyRange, - rowFilter, - newLimit, - countCQL3Rows, - isPaging); - } - - public int limit() - { - return maxResults; - } - - public boolean countCQL3Rows() - { - return countCQL3Rows; - } - - public List executeLocally() - { - ColumnFamilyStore cfs = Keyspace.open(keyspace).getColumnFamilyStore(columnFamily); - - ExtendedFilter exFilter = cfs.makeExtendedFilter(keyRange, predicate, rowFilter, maxResults, countCQL3Rows, isPaging, timestamp); - if (cfs.indexManager.hasIndexFor(rowFilter)) - return cfs.search(exFilter); - else - return cfs.getRangeSlice(exFilter); - } - - @Override - public String toString() - { - return Objects.toStringHelper(this) - .add("keyspace", keyspace) - .add("columnFamily", columnFamily) - .add("predicate", predicate) - .add("keyRange", keyRange) - .add("rowFilter", rowFilter) - .add("maxResults", maxResults) - .add("counterCQL3Rows", countCQL3Rows) - .add("timestamp", timestamp) - .toString(); - } -} - -class RangeSliceCommandSerializer implements IVersionedSerializer -{ - public void serialize(RangeSliceCommand sliceCommand, DataOutputPlus out, int version) throws IOException - { - out.writeUTF(sliceCommand.keyspace); - out.writeUTF(sliceCommand.columnFamily); - out.writeLong(sliceCommand.timestamp); - - CFMetaData metadata = Schema.instance.getCFMetaData(sliceCommand.keyspace, sliceCommand.columnFamily); - - metadata.comparator.diskAtomFilterSerializer().serialize(sliceCommand.predicate, out, version); - - if (sliceCommand.rowFilter == null) - { - out.writeInt(0); - } - else - { - out.writeInt(sliceCommand.rowFilter.size()); - for (IndexExpression expr : sliceCommand.rowFilter) - { - expr.writeTo(out); - } - } - MessagingService.validatePartitioner(sliceCommand.keyRange); - AbstractBounds.rowPositionSerializer.serialize(sliceCommand.keyRange, out, version); - out.writeInt(sliceCommand.maxResults); - out.writeBoolean(sliceCommand.countCQL3Rows); - out.writeBoolean(sliceCommand.isPaging); - } - - public RangeSliceCommand deserialize(DataInput in, int version) throws IOException - { - String keyspace = in.readUTF(); - String columnFamily = in.readUTF(); - long timestamp = in.readLong(); - - CFMetaData metadata = Schema.instance.getCFMetaData(keyspace, columnFamily); - if (metadata == null) - { - String message = String.format("Got range slice command for nonexistent table %s.%s. If the table was just " + - "created, this is likely due to the schema not being fully propagated. Please wait for schema " + - "agreement on table creation." , keyspace, columnFamily); - throw new UnknownColumnFamilyException(message, null); - } - - IDiskAtomFilter predicate = metadata.comparator.diskAtomFilterSerializer().deserialize(in, version); - - List rowFilter; - int filterCount = in.readInt(); - rowFilter = new ArrayList<>(filterCount); - for (int i = 0; i < filterCount; i++) - { - rowFilter.add(IndexExpression.readFrom(in)); - } - AbstractBounds range = AbstractBounds.rowPositionSerializer.deserialize(in, MessagingService.globalPartitioner(), version); - - int maxResults = in.readInt(); - boolean countCQL3Rows = in.readBoolean(); - boolean isPaging = in.readBoolean(); - return new RangeSliceCommand(keyspace, columnFamily, timestamp, predicate, range, rowFilter, maxResults, countCQL3Rows, isPaging); - } - - public long serializedSize(RangeSliceCommand rsc, int version) - { - long size = TypeSizes.NATIVE.sizeof(rsc.keyspace); - size += TypeSizes.NATIVE.sizeof(rsc.columnFamily); - size += TypeSizes.NATIVE.sizeof(rsc.timestamp); - - CFMetaData metadata = Schema.instance.getCFMetaData(rsc.keyspace, rsc.columnFamily); - - IDiskAtomFilter filter = rsc.predicate; - - size += metadata.comparator.diskAtomFilterSerializer().serializedSize(filter, version); - - if (rsc.rowFilter == null) - { - size += TypeSizes.NATIVE.sizeof(0); - } - else - { - size += TypeSizes.NATIVE.sizeof(rsc.rowFilter.size()); - for (IndexExpression expr : rsc.rowFilter) - { - size += TypeSizes.NATIVE.sizeofWithShortLength(expr.column); - size += TypeSizes.NATIVE.sizeof(expr.operator.ordinal()); - size += TypeSizes.NATIVE.sizeofWithShortLength(expr.value); - } - } - size += AbstractBounds.rowPositionSerializer.serializedSize(rsc.keyRange, version); - size += TypeSizes.NATIVE.sizeof(rsc.maxResults); - size += TypeSizes.NATIVE.sizeof(rsc.countCQL3Rows); - size += TypeSizes.NATIVE.sizeof(rsc.isPaging); - return size; - } -} diff --git a/src/java/org/apache/cassandra/db/RangeSliceReply.java b/src/java/org/apache/cassandra/db/RangeSliceReply.java deleted file mode 100644 index ed1f523a64..0000000000 --- a/src/java/org/apache/cassandra/db/RangeSliceReply.java +++ /dev/null @@ -1,92 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.apache.cassandra.db; - -import java.io.DataInput; -import java.io.DataInputStream; -import java.io.IOException; -import java.util.ArrayList; -import java.util.List; - -import org.apache.commons.lang3.StringUtils; - -import org.apache.cassandra.io.IVersionedSerializer; -import org.apache.cassandra.io.util.DataOutputPlus; -import org.apache.cassandra.io.util.FastByteArrayInputStream; -import org.apache.cassandra.net.MessageOut; -import org.apache.cassandra.net.MessagingService; - -public class RangeSliceReply -{ - public static final RangeSliceReplySerializer serializer = new RangeSliceReplySerializer(); - - public final List rows; - - public RangeSliceReply(List rows) - { - this.rows = rows; - } - - public MessageOut createMessage() - { - return new MessageOut(MessagingService.Verb.REQUEST_RESPONSE, this, serializer); - } - - @Override - public String toString() - { - return "RangeSliceReply{" + - "rows=" + StringUtils.join(rows, ",") + - '}'; - } - - public static RangeSliceReply read(byte[] body, int version) throws IOException - { - try (DataInputStream dis = new DataInputStream(new FastByteArrayInputStream(body))) - { - return serializer.deserialize(dis, version); - } - } - - private static class RangeSliceReplySerializer implements IVersionedSerializer - { - public void serialize(RangeSliceReply rsr, DataOutputPlus out, int version) throws IOException - { - out.writeInt(rsr.rows.size()); - for (Row row : rsr.rows) - Row.serializer.serialize(row, out, version); - } - - public RangeSliceReply deserialize(DataInput in, int version) throws IOException - { - int rowCount = in.readInt(); - List rows = new ArrayList(rowCount); - for (int i = 0; i < rowCount; i++) - rows.add(Row.serializer.deserialize(in, version)); - return new RangeSliceReply(rows); - } - - public long serializedSize(RangeSliceReply rsr, int version) - { - int size = TypeSizes.NATIVE.sizeof(rsr.rows.size()); - for (Row row : rsr.rows) - size += Row.serializer.serializedSize(row, version); - return size; - } - } -} diff --git a/src/java/org/apache/cassandra/db/RangeTombstone.java b/src/java/org/apache/cassandra/db/RangeTombstone.java index da483fca02..3373afad86 100644 --- a/src/java/org/apache/cassandra/db/RangeTombstone.java +++ b/src/java/org/apache/cassandra/db/RangeTombstone.java @@ -19,12 +19,12 @@ package org.apache.cassandra.db; import java.io.DataInput; import java.io.IOException; +import java.nio.ByteBuffer; import java.security.MessageDigest; import java.util.*; import org.apache.cassandra.config.CFMetaData; -import org.apache.cassandra.db.composites.CType; -import org.apache.cassandra.db.composites.Composite; +import org.apache.cassandra.db.marshal.AbstractType; import org.apache.cassandra.io.ISSTableSerializer; import org.apache.cassandra.io.sstable.format.Version; import org.apache.cassandra.io.util.DataOutputBuffer; @@ -32,350 +32,142 @@ import org.apache.cassandra.io.util.DataOutputPlus; import org.apache.cassandra.serializers.MarshalException; import org.apache.cassandra.utils.Interval; -public class RangeTombstone extends Interval implements OnDiskAtom +/** + * A range tombstone is a tombstone that covers a slice/range of rows. + *

+ * Note that in most of the storage engine, a range tombstone is actually represented by its separated + * opening and closing bound, see {@link RangeTombstoneMarker}. So in practice, this is only used when + * full partitions are materialized in memory in a {@code Partition} object, and more precisely through + * the use of a {@code RangeTombstoneList} in a {@code DeletionInfo} object. + */ +public class RangeTombstone { - public RangeTombstone(Composite start, Composite stop, long markedForDeleteAt, int localDeletionTime) - { - this(start, stop, new DeletionTime(markedForDeleteAt, localDeletionTime)); - } + private final Slice slice; + private final DeletionTime deletion; - public RangeTombstone(Composite start, Composite stop, DeletionTime delTime) + public RangeTombstone(Slice slice, DeletionTime deletion) { - super(start, stop, delTime); - } - - public Composite name() - { - return min; - } - - public int getLocalDeletionTime() - { - return data.localDeletionTime; - } - - public long timestamp() - { - return data.markedForDeleteAt; - } - - public void validateFields(CFMetaData metadata) throws MarshalException - { - metadata.comparator.validate(min); - metadata.comparator.validate(max); - } - - public void updateDigest(MessageDigest digest) - { - digest.update(min.toByteBuffer().duplicate()); - digest.update(max.toByteBuffer().duplicate()); - - try (DataOutputBuffer buffer = new DataOutputBuffer()) - { - buffer.writeLong(data.markedForDeleteAt); - digest.update(buffer.getData(), 0, buffer.getLength()); - } - catch (IOException e) - { - throw new RuntimeException(e); - } + this.slice = slice; + this.deletion = deletion.takeAlias(); } /** - * This tombstone supersedes another one if it is more recent and cover a - * bigger range than rt. + * The slice of rows that is deleted by this range tombstone. + * + * @return the slice of rows that is deleted by this range tombstone. */ - public boolean supersedes(RangeTombstone rt, Comparator comparator) + public Slice deletedSlice() { - if (rt.data.markedForDeleteAt > data.markedForDeleteAt) + return slice; + } + + /** + * The deletion time for this (range) tombstone. + * + * @return the deletion time for this range tombstone. + */ + public DeletionTime deletionTime() + { + return deletion; + } + + @Override + public boolean equals(Object other) + { + if(!(other instanceof RangeTombstone)) return false; - return comparator.compare(min, rt.min) <= 0 && comparator.compare(max, rt.max) >= 0; + RangeTombstone that = (RangeTombstone)other; + return this.deletedSlice().equals(that.deletedSlice()) + && this.deletionTime().equals(that.deletionTime()); } - public boolean includes(Comparator comparator, Composite name) + @Override + public int hashCode() { - return comparator.compare(name, min) >= 0 && comparator.compare(name, max) <= 0; + return Objects.hash(deletedSlice(), deletionTime()); } /** - * Tracks opened RangeTombstones when iterating over a partition. + * The bound of a range tombstone. *

- * This tracker must be provided all the atoms of a given partition in - * order (to the {@code update} method). Given this, it keeps enough - * information to be able to decide if one of an atom is deleted (shadowed) - * by a previously open RT. One the tracker can prove a given range - * tombstone cannot be useful anymore (that is, as soon as we've seen an - * atom that is after the end of that RT), it discards this RT. In other - * words, the maximum memory used by this object should be proportional to - * the maximum number of RT that can be simultaneously open (and this - * should fairly low in practice). + * This is the same than for a slice but it includes "boundaries" between ranges. A boundary simply condensed + * a close and an opening "bound" into a single object. There is 2 main reasons for these "shortcut" boundaries: + * 1) When merging multiple iterators having range tombstones (that are represented by their start and end markers), + * we need to know when a range is close on an iterator, if it is reopened right away. Otherwise, we cannot + * easily produce the markers on the merged iterators within risking to fail the sorting guarantees of an + * iterator. See this comment for more details: https://goo.gl/yyB5mR. + * 2) This saves some storage space. */ - public static class Tracker + public static class Bound extends Slice.Bound { - private final Comparator comparator; + public static final Serializer serializer = new Serializer(); - // A list the currently open RTs. We keep the list sorted in order of growing end bounds as for a - // new atom, this allows to efficiently find the RTs that are now useless (if any). Also note that because - // atom are passed to the tracker in order, any RT that is tracked can be assumed as opened, i.e. we - // never have to test the RTs start since it's always assumed to be less than what we have. - // Also note that this will store expired RTs (#7810). Those will be of type ExpiredRangeTombstone and - // will be ignored by writeOpenedMarker. - private final List openedTombstones = new LinkedList(); - - // Total number of atoms written by writeOpenedMarker(). - private int atomCount; - - /** - * Creates a new tracker given the table comparator. - * - * @param comparator the comparator for the table this will track atoms - * for. The tracker assumes that atoms will be later provided to the - * tracker in {@code comparator} order. - */ - public Tracker(Comparator comparator) + public Bound(Kind kind, ByteBuffer[] values) { - this.comparator = comparator; + super(kind, values); } - /** - * Computes the RangeTombstone that are needed at the beginning of an index - * block starting with {@code firstColumn}. - * - * @return the total serialized size of said tombstones and write them to - * {@code out} it if isn't null. - */ - public long writeOpenedMarker(OnDiskAtom firstColumn, DataOutputPlus out, OnDiskAtom.Serializer atomSerializer) throws IOException + public static RangeTombstone.Bound inclusiveOpen(boolean reversed, ByteBuffer[] boundValues) { - long size = 0; - if (openedTombstones.isEmpty()) - return size; + return new Bound(reversed ? Kind.INCL_END_BOUND : Kind.INCL_START_BOUND, boundValues); + } - /* - * Compute the markers that needs to be written at the beginning of - * this block. We need to write one if it is the more recent - * (opened) tombstone for at least some part of its range. - */ - List toWrite = new LinkedList(); - outer: - for (RangeTombstone tombstone : openedTombstones) + public static RangeTombstone.Bound exclusiveOpen(boolean reversed, ByteBuffer[] boundValues) + { + return new Bound(reversed ? Kind.EXCL_END_BOUND : Kind.EXCL_START_BOUND, boundValues); + } + + public static RangeTombstone.Bound inclusiveClose(boolean reversed, ByteBuffer[] boundValues) + { + return new Bound(reversed ? Kind.INCL_START_BOUND : Kind.INCL_END_BOUND, boundValues); + } + + public static RangeTombstone.Bound exclusiveClose(boolean reversed, ByteBuffer[] boundValues) + { + return new Bound(reversed ? Kind.EXCL_START_BOUND : Kind.EXCL_END_BOUND, boundValues); + } + + public static RangeTombstone.Bound inclusiveCloseExclusiveOpen(boolean reversed, ByteBuffer[] boundValues) + { + return new Bound(reversed ? Kind.EXCL_END_INCL_START_BOUNDARY : Kind.INCL_END_EXCL_START_BOUNDARY, boundValues); + } + + public static RangeTombstone.Bound exclusiveCloseInclusiveOpen(boolean reversed, ByteBuffer[] boundValues) + { + return new Bound(reversed ? Kind.INCL_END_EXCL_START_BOUNDARY : Kind.EXCL_END_INCL_START_BOUNDARY, boundValues); + } + + @Override + public Bound withNewKind(Kind kind) + { + return new Bound(kind, values); + } + + public static class Serializer + { + public void serialize(RangeTombstone.Bound bound, DataOutputPlus out, int version, List> types) throws IOException { - // If the first column is outside the range, skip it (in case update() hasn't been called yet) - if (comparator.compare(firstColumn.name(), tombstone.max) > 0) - continue; - - if (tombstone instanceof ExpiredRangeTombstone) - continue; - - RangeTombstone updated = new RangeTombstone(firstColumn.name(), tombstone.max, tombstone.data); - - Iterator iter = toWrite.iterator(); - while (iter.hasNext()) - { - RangeTombstone other = iter.next(); - if (other.supersedes(updated, comparator)) - break outer; - if (updated.supersedes(other, comparator)) - iter.remove(); - } - toWrite.add(tombstone); + out.writeByte(bound.kind().ordinal()); + out.writeShort(bound.size()); + ClusteringPrefix.serializer.serializeValuesWithoutSize(bound, out, version, types); } - for (RangeTombstone tombstone : toWrite) + public long serializedSize(RangeTombstone.Bound bound, int version, List> types, TypeSizes sizes) { - size += atomSerializer.serializedSizeForSSTable(tombstone); - atomCount++; - if (out != null) - atomSerializer.serializeForSSTable(tombstone, out); - } - return size; - } - - /** - * The total number of atoms written by calls to the method {@link #writeOpenedMarker}. - */ - public int writtenAtom() - { - return atomCount; - } - - /** - * Update this tracker given an {@code atom}. - *

- * This method first test if some range tombstone can be discarded due - * to the knowledge of that new atom. Then, if it's a range tombstone, - * it adds it to the tracker. - *

- * Note that this method should be called on *every* atom of a partition for - * the tracker to work as efficiently as possible (#9486). - */ - public void update(OnDiskAtom atom, boolean isExpired) - { - // Get rid of now useless RTs - ListIterator iterator = openedTombstones.listIterator(); - while (iterator.hasNext()) - { - // If this tombstone stops before the new atom, it is now useless since it cannot cover this or any future - // atoms. Otherwise, if a RT ends after the new atom, then we know that's true of any following atom too - // since maxOrderingSet is sorted by end bounds - RangeTombstone t = iterator.next(); - if (comparator.compare(atom.name(), t.max) > 0) - { - iterator.remove(); - } - else - { - // If the atom is a RT, we'll add it next and for that we want to start by looking at the atom we just - // returned, so rewind the iterator. - iterator.previous(); - break; - } + return 1 // kind ordinal + + sizes.sizeof((short)bound.size()) + + ClusteringPrefix.serializer.valuesWithoutSizeSerializedSize(bound, version, types, sizes); } - // If it's a RT, adds it. - if (atom instanceof RangeTombstone) + public Kind deserialize(DataInput in, int version, List> types, Writer writer) throws IOException { - RangeTombstone toAdd = (RangeTombstone)atom; - if (isExpired) - toAdd = new ExpiredRangeTombstone(toAdd); - - // We want to maintain openedTombstones in end bounds order so we find where to insert the new element - // and add it. While doing so, we also check if that new tombstone fully shadow or is fully shadowed - // by an existing tombstone so we avoid tracking more tombstone than necessary (and we know this will - // at least happend for start-of-index-block repeated range tombstones). - while (iterator.hasNext()) - { - RangeTombstone existing = iterator.next(); - int cmp = comparator.compare(toAdd.max, existing.max); - if (cmp > 0) - { - // the new one covers more than the existing one. If the new one happens to also supersedes - // the existing one, remove the existing one. In any case, we're not done yet. - if (toAdd.data.supersedes(existing.data)) - iterator.remove(); - } - else - { - // the new one is included in the existing one. If the new one supersedes the existing one, - // then we add the new one (and if the new one ends like the existing one, we can actually remove - // the existing one), otherwise we can actually ignore it. In any case, we're done. - if (toAdd.data.supersedes(existing.data)) - { - if (cmp == 0) - iterator.set(toAdd); - else - insertBefore(toAdd, iterator); - } - return; - } - } - // If we reach here, either we had no tombstones and the new one ends after all existing ones. - iterator.add(toAdd); + Kind kind = Kind.values()[in.readByte()]; + writer.writeBoundKind(kind); + int size = in.readUnsignedShort(); + ClusteringPrefix.serializer.deserializeValuesWithoutSize(in, size, version, types, writer); + return kind; } } - - /** - * Adds the provided {@code tombstone} _before_ the last element returned by {@code iterator.next()}. - *

- * This method assumes that {@code iterator.next()} has been called prior to this method call, i.e. that - * {@code iterator.hasPrevious() == true}. - */ - private static void insertBefore(RangeTombstone tombstone, ListIterator iterator) - { - assert iterator.hasPrevious(); - iterator.previous(); - iterator.add(tombstone); - iterator.next(); - } - - /** - * Tests if the provided column is deleted by one of the tombstone - * tracked by this tracker. - *

- * This method should be called on columns in the same order than for the update() - * method. Note that this method does not update the tracker so the update() method - * should still be called on {@code column} (it doesn't matter if update is called - * before or after this call). - */ - public boolean isDeleted(Cell cell) - { - // We know every tombstone kept are "open", start before the column. So the - // column is deleted if any of the tracked tombstone ends after the column - // (this will be the case of every RT if update() has been called before this - // method, but we might have a few RT to skip otherwise) and the RT deletion is - // actually more recent than the column timestamp. - for (RangeTombstone tombstone : openedTombstones) - { - if (comparator.compare(cell.name(), tombstone.max) <= 0 - && tombstone.timestamp() >= cell.timestamp()) - return true; - } - return false; - } - - /** - * The tracker needs to track expired range tombstone but keep tracks that they are - * expired, so this is what this class is used for. - */ - private static class ExpiredRangeTombstone extends RangeTombstone - { - private ExpiredRangeTombstone(RangeTombstone tombstone) - { - super(tombstone.min, tombstone.max, tombstone.data); - } - } - } - - public static class Serializer implements ISSTableSerializer - { - private final CType type; - - public Serializer(CType type) - { - this.type = type; - } - - public void serializeForSSTable(RangeTombstone t, DataOutputPlus out) throws IOException - { - type.serializer().serialize(t.min, out); - out.writeByte(ColumnSerializer.RANGE_TOMBSTONE_MASK); - type.serializer().serialize(t.max, out); - DeletionTime.serializer.serialize(t.data, out); - } - - public RangeTombstone deserializeFromSSTable(DataInput in, Version version) throws IOException - { - Composite min = type.serializer().deserialize(in); - - int b = in.readUnsignedByte(); - assert (b & ColumnSerializer.RANGE_TOMBSTONE_MASK) != 0; - return deserializeBody(in, min, version); - } - - public RangeTombstone deserializeBody(DataInput in, Composite min, Version version) throws IOException - { - Composite max = type.serializer().deserialize(in); - DeletionTime dt = DeletionTime.serializer.deserialize(in); - // If the max equals the min.end(), we can avoid keeping an extra ByteBuffer in memory by using - // min.end() instead of max - Composite minEnd = min.end(); - max = minEnd.equals(max) ? minEnd : max; - return new RangeTombstone(min, max, dt); - } - - public void skipBody(DataInput in, Version version) throws IOException - { - type.serializer().skip(in); - DeletionTime.serializer.skip(in); - } - - public long serializedSizeForSSTable(RangeTombstone t) - { - TypeSizes typeSizes = TypeSizes.NATIVE; - return type.serializer().serializedSize(t.min, typeSizes) - + 1 // serialization flag - + type.serializer().serializedSize(t.max, typeSizes) - + DeletionTime.serializer.serializedSize(t.data, typeSizes); - } } } diff --git a/src/java/org/apache/cassandra/db/RangeTombstoneList.java b/src/java/org/apache/cassandra/db/RangeTombstoneList.java index 37f1ef41d3..0c27bc4184 100644 --- a/src/java/org/apache/cassandra/db/RangeTombstoneList.java +++ b/src/java/org/apache/cassandra/db/RangeTombstoneList.java @@ -20,9 +20,7 @@ package org.apache.cassandra.db; import java.io.DataInput; import java.io.IOException; import java.nio.ByteBuffer; -import java.security.MessageDigest; import java.util.Arrays; -import java.util.Comparator; import java.util.Iterator; import com.google.common.collect.AbstractIterator; @@ -31,12 +29,9 @@ import org.slf4j.Logger; import org.slf4j.LoggerFactory; import org.apache.cassandra.cache.IMeasurableMemory; -import org.apache.cassandra.db.composites.CType; -import org.apache.cassandra.db.composites.CellName; -import org.apache.cassandra.db.composites.Composite; +import org.apache.cassandra.db.rows.*; import org.apache.cassandra.io.IVersionedSerializer; import org.apache.cassandra.io.util.DataOutputPlus; -import org.apache.cassandra.net.MessagingService; import org.apache.cassandra.utils.ObjectSizes; import org.apache.cassandra.utils.memory.AbstractAllocator; @@ -62,19 +57,19 @@ public class RangeTombstoneList implements Iterable, IMeasurable private static long EMPTY_SIZE = ObjectSizes.measure(new RangeTombstoneList(null, 0)); - private final Comparator comparator; + private final ClusteringComparator comparator; // Note: we don't want to use a List for the markedAts and delTimes to avoid boxing. We could // use a List for starts and ends, but having arrays everywhere is almost simpler. - private Composite[] starts; - private Composite[] ends; + private Slice.Bound[] starts; + private Slice.Bound[] ends; private long[] markedAts; private int[] delTimes; private long boundaryHeapSize; private int size; - private RangeTombstoneList(Comparator comparator, Composite[] starts, Composite[] ends, long[] markedAts, int[] delTimes, long boundaryHeapSize, int size) + private RangeTombstoneList(ClusteringComparator comparator, Slice.Bound[] starts, Slice.Bound[] ends, long[] markedAts, int[] delTimes, long boundaryHeapSize, int size) { assert starts.length == ends.length && starts.length == markedAts.length && starts.length == delTimes.length; this.comparator = comparator; @@ -86,9 +81,9 @@ public class RangeTombstoneList implements Iterable, IMeasurable this.boundaryHeapSize = boundaryHeapSize; } - public RangeTombstoneList(Comparator comparator, int capacity) + public RangeTombstoneList(ClusteringComparator comparator, int capacity) { - this(comparator, new Composite[capacity], new Composite[capacity], new long[capacity], new int[capacity], 0, 0); + this(comparator, new Slice.Bound[capacity], new Slice.Bound[capacity], new long[capacity], new int[capacity], 0, 0); } public boolean isEmpty() @@ -101,7 +96,7 @@ public class RangeTombstoneList implements Iterable, IMeasurable return size; } - public Comparator comparator() + public ClusteringComparator comparator() { return comparator; } @@ -119,27 +114,36 @@ public class RangeTombstoneList implements Iterable, IMeasurable public RangeTombstoneList copy(AbstractAllocator allocator) { RangeTombstoneList copy = new RangeTombstoneList(comparator, - new Composite[size], - new Composite[size], - Arrays.copyOf(markedAts, size), - Arrays.copyOf(delTimes, size), - boundaryHeapSize, size); + new Slice.Bound[size], + new Slice.Bound[size], + Arrays.copyOf(markedAts, size), + Arrays.copyOf(delTimes, size), + boundaryHeapSize, size); for (int i = 0; i < size; i++) { - assert !(starts[i] instanceof AbstractNativeCell || ends[i] instanceof AbstractNativeCell); //this should never happen - - copy.starts[i] = starts[i].copy(null, allocator); - copy.ends[i] = ends[i].copy(null, allocator); + copy.starts[i] = clone(starts[i], allocator); + copy.ends[i] = clone(ends[i], allocator); } return copy; } + private static Slice.Bound clone(Slice.Bound bound, AbstractAllocator allocator) + { + ByteBuffer[] values = new ByteBuffer[bound.size()]; + for (int i = 0; i < values.length; i++) + values[i] = allocator.clone(bound.get(i)); + return new Slice.Bound(bound.kind(), values); + } + public void add(RangeTombstone tombstone) { - add(tombstone.min, tombstone.max, tombstone.data.markedForDeleteAt, tombstone.data.localDeletionTime); + add(tombstone.deletedSlice().start(), + tombstone.deletedSlice().end(), + tombstone.deletionTime().markedForDeleteAt(), + tombstone.deletionTime().localDeletionTime()); } /** @@ -148,7 +152,7 @@ public class RangeTombstoneList implements Iterable, IMeasurable * This method will be faster if the new tombstone sort after all the currently existing ones (this is a common use case), * but it doesn't assume it. */ - public void add(Composite start, Composite end, long markedAt, int delTime) + public void add(Slice.Bound start, Slice.Bound end, long markedAt, int delTime) { if (isEmpty()) { @@ -215,7 +219,7 @@ public class RangeTombstoneList implements Iterable, IMeasurable int j = 0; while (i < size && j < tombstones.size) { - if (comparator.compare(tombstones.starts[j], ends[i]) <= 0) + if (comparator.compare(tombstones.starts[j], ends[i]) < 0) { insertFrom(i, tombstones.starts[j], tombstones.ends[j], tombstones.markedAts[j], tombstones.delTimes[j]); j++; @@ -235,34 +239,26 @@ public class RangeTombstoneList implements Iterable, IMeasurable * Returns whether the given name/timestamp pair is deleted by one of the tombstone * of this RangeTombstoneList. */ - public boolean isDeleted(Cell cell) + public boolean isDeleted(Clustering clustering, Cell cell) { - int idx = searchInternal(cell.name(), 0); + int idx = searchInternal(clustering, 0, size); // No matter what the counter cell's timestamp is, a tombstone always takes precedence. See CASSANDRA-7346. - return idx >= 0 && (cell instanceof CounterCell || markedAts[idx] >= cell.timestamp()); - } - - /** - * Returns a new {@link InOrderTester}. - */ - InOrderTester inOrderTester() - { - return new InOrderTester(); + return idx >= 0 && (cell.isCounterCell() || markedAts[idx] >= cell.livenessInfo().timestamp()); } /** * Returns the DeletionTime for the tombstone overlapping {@code name} (there can't be more than one), * or null if {@code name} is not covered by any tombstone. */ - public DeletionTime searchDeletionTime(Composite name) + public DeletionTime searchDeletionTime(Clustering name) { - int idx = searchInternal(name, 0); - return idx < 0 ? null : new DeletionTime(markedAts[idx], delTimes[idx]); + int idx = searchInternal(name, 0, size); + return idx < 0 ? null : new SimpleDeletionTime(markedAts[idx], delTimes[idx]); } - public RangeTombstone search(Composite name) + public RangeTombstone search(Clustering name) { - int idx = searchInternal(name, 0); + int idx = searchInternal(name, 0, size); return idx < 0 ? null : rangeTombstone(idx); } @@ -270,20 +266,15 @@ public class RangeTombstoneList implements Iterable, IMeasurable * Return is the index of the range covering name if name is covered. If the return idx is negative, * no range cover name and -idx-1 is the index of the first range whose start is greater than name. */ - private int searchInternal(Composite name, int startIdx) + private int searchInternal(ClusteringPrefix name, int startIdx, int endIdx) { if (isEmpty()) return -1; - int pos = Arrays.binarySearch(starts, startIdx, size, name, comparator); + int pos = Arrays.binarySearch(starts, startIdx, endIdx, name, comparator); if (pos >= 0) { - // We're exactly on an interval start. The one subtility is that we need to check if - // the previous is not equal to us and doesn't have a higher marked at - if (pos > 0 && comparator.compare(name, ends[pos-1]) == 0 && markedAts[pos-1] > markedAts[pos]) - return pos-1; - else - return pos; + return pos; } else { @@ -308,93 +299,94 @@ public class RangeTombstoneList implements Iterable, IMeasurable return dataSize; } - public long minMarkedAt() - { - long min = Long.MAX_VALUE; - for (int i = 0; i < size; i++) - min = Math.min(min, markedAts[i]); - return min; - } - - public long maxMarkedAt() - { - long max = Long.MIN_VALUE; - for (int i = 0; i < size; i++) - max = Math.max(max, markedAts[i]); - return max; - } - public void updateAllTimestamp(long timestamp) { for (int i = 0; i < size; i++) markedAts[i] = timestamp; } - /** - * Removes all range tombstones whose local deletion time is older than gcBefore. - */ - public void purge(int gcBefore) - { - int j = 0; - for (int i = 0; i < size; i++) - { - if (delTimes[i] >= gcBefore) - setInternal(j++, starts[i], ends[i], markedAts[i], delTimes[i]); - } - size = j; - } - - /** - * Returns whether {@code purge(gcBefore)} would remove something or not. - */ - public boolean hasPurgeableTombstones(int gcBefore) - { - for (int i = 0; i < size; i++) - { - if (delTimes[i] < gcBefore) - return true; - } - return false; - } - private RangeTombstone rangeTombstone(int idx) { - return new RangeTombstone(starts[idx], ends[idx], markedAts[idx], delTimes[idx]); + return new RangeTombstone(Slice.make(starts[idx], ends[idx]), new SimpleDeletionTime(markedAts[idx], delTimes[idx])); + } + + private RangeTombstone rangeTombstoneWithNewStart(int idx, Slice.Bound newStart) + { + return new RangeTombstone(Slice.make(newStart, ends[idx]), new SimpleDeletionTime(markedAts[idx], delTimes[idx])); + } + + private RangeTombstone rangeTombstoneWithNewEnd(int idx, Slice.Bound newEnd) + { + return new RangeTombstone(Slice.make(starts[idx], newEnd), new SimpleDeletionTime(markedAts[idx], delTimes[idx])); + } + + private RangeTombstone rangeTombstoneWithNewBounds(int idx, Slice.Bound newStart, Slice.Bound newEnd) + { + return new RangeTombstone(Slice.make(newStart, newEnd), new SimpleDeletionTime(markedAts[idx], delTimes[idx])); } public Iterator iterator() { - return new AbstractIterator() - { - private int idx; - - protected RangeTombstone computeNext() - { - if (idx >= size) - return endOfData(); - - return rangeTombstone(idx++); - } - }; + return iterator(false); } - public Iterator iterator(Composite from, Composite till) + public Iterator iterator(boolean reversed) { - int startIdx = from.isEmpty() ? 0 : searchInternal(from, 0); + return reversed + ? new AbstractIterator() + { + private int idx = size - 1; + + protected RangeTombstone computeNext() + { + if (idx < 0) + return endOfData(); + + return rangeTombstone(idx--); + } + } + : new AbstractIterator() + { + private int idx; + + protected RangeTombstone computeNext() + { + if (idx >= size) + return endOfData(); + + return rangeTombstone(idx++); + } + }; + } + + public Iterator iterator(final Slice slice, boolean reversed) + { + return reversed ? reverseIterator(slice) : forwardIterator(slice); + } + + private Iterator forwardIterator(final Slice slice) + { + int startIdx = slice.start() == Slice.Bound.BOTTOM ? 0 : searchInternal(slice.start(), 0, size); final int start = startIdx < 0 ? -startIdx-1 : startIdx; if (start >= size) return Iterators.emptyIterator(); - int finishIdx = till.isEmpty() ? size : searchInternal(till, start); - // if stopIdx is the first range after 'till' we care only until the previous range + int finishIdx = slice.end() == Slice.Bound.TOP ? size - 1 : searchInternal(slice.end(), start, size); + // if stopIdx is the first range after 'slice.end()' we care only until the previous range final int finish = finishIdx < 0 ? -finishIdx-2 : finishIdx; - // Note: the following is true because we know 'from' is before 'till' in sorted order. if (start > finish) return Iterators.emptyIterator(); - else if (start == finish) - return Iterators.singletonIterator(rangeTombstone(start)); + + if (start == finish) + { + // We want to make sure the range are stricly included within the queried slice as this + // make it easier to combine things when iterating over successive slices. + Slice.Bound s = comparator.compare(starts[start], slice.start()) < 0 ? slice.start() : starts[start]; + Slice.Bound e = comparator.compare(slice.end(), ends[start]) < 0 ? slice.end() : ends[start]; + return Iterators.singletonIterator(rangeTombstoneWithNewBounds(start, s, e)); + } return new AbstractIterator() { @@ -405,77 +397,64 @@ public class RangeTombstoneList implements Iterable, IMeasurable if (idx >= size || idx > finish) return endOfData(); + // We want to make sure the range are stricly included within the queried slice as this + // make it easier to combine things when iterating over successive slices. This means that + // for the first and last range we might have to "cut" the range returned. + if (idx == start && comparator.compare(starts[idx], slice.start()) < 0) + return rangeTombstoneWithNewStart(idx++, slice.start()); + if (idx == finish && comparator.compare(slice.end(), ends[idx]) < 0) + return rangeTombstoneWithNewEnd(idx++, slice.end()); return rangeTombstone(idx++); } }; } - /** - * Evaluates a diff between superset (known to be all merged tombstones) and this list for read repair - * - * @return null if there is no difference - */ - public RangeTombstoneList diff(RangeTombstoneList superset) + private Iterator reverseIterator(final Slice slice) { - if (isEmpty()) - return superset; + int startIdx = slice.end() == Slice.Bound.TOP ? 0 : searchInternal(slice.end(), 0, size); + // if startIdx is the first range after 'slice.end()' we care only until the previous range + final int start = startIdx < 0 ? -startIdx-2 : startIdx; - RangeTombstoneList diff = null; + if (start >= size) + return Iterators.emptyIterator(); - int j = 0; // index to iterate through our own list - for (int i = 0; i < superset.size; i++) + int finishIdx = slice.start() == Slice.Bound.BOTTOM ? 0 : searchInternal(slice.start(), 0, start); + // if stopIdx is the first range after 'slice.end()' we care only until the previous range + final int finish = finishIdx < 0 ? -finishIdx-1 : finishIdx; + + if (start < finish) + return Iterators.emptyIterator(); + + if (start == finish) { - // we can assume that this list is a subset of the superset list - while (j < size && comparator.compare(starts[j], superset.starts[i]) < 0) - j++; - - if (j >= size) - { - // we're at the end of our own list, add the remainder of the superset to the diff - if (i < superset.size) - { - if (diff == null) - diff = new RangeTombstoneList(comparator, superset.size - i); - - for(int k = i; k < superset.size; k++) - diff.add(superset.starts[k], superset.ends[k], superset.markedAts[k], superset.delTimes[k]); - } - return diff; - } - - // we don't care about local deletion time here, because it doesn't matter for read repair - if (!starts[j].equals(superset.starts[i]) - || !ends[j].equals(superset.ends[i]) - || markedAts[j] != superset.markedAts[i]) - { - if (diff == null) - diff = new RangeTombstoneList(comparator, Math.min(8, superset.size - i)); - diff.add(superset.starts[i], superset.ends[i], superset.markedAts[i], superset.delTimes[i]); - } + // We want to make sure the range are stricly included within the queried slice as this + // make it easier to combine things when iterator over successive slices. + Slice.Bound s = comparator.compare(starts[start], slice.start()) < 0 ? slice.start() : starts[start]; + Slice.Bound e = comparator.compare(slice.end(), ends[start]) < 0 ? slice.end() : ends[start]; + return Iterators.singletonIterator(rangeTombstoneWithNewBounds(start, s, e)); } - return diff; - } - - /** - * Calculates digest for triggering read repair on mismatch - */ - public void updateDigest(MessageDigest digest) - { - ByteBuffer longBuffer = ByteBuffer.allocate(8); - for (int i = 0; i < size; i++) + return new AbstractIterator() { - for (int j = 0; j < starts[i].size(); j++) - digest.update(starts[i].get(j).duplicate()); - for (int j = 0; j < ends[i].size(); j++) - digest.update(ends[i].get(j).duplicate()); + private int idx = start; - longBuffer.putLong(0, markedAts[i]); - digest.update(longBuffer.array(), 0, 8); - } + protected RangeTombstone computeNext() + { + if (idx < 0 || idx < finish) + return endOfData(); + + // We want to make sure the range are stricly included within the queried slice as this + // make it easier to combine things when iterator over successive slices. This means that + // for the first and last range we might have to "cut" the range returned. + if (idx == start && comparator.compare(slice.end(), ends[idx]) < 0) + return rangeTombstoneWithNewEnd(idx--, slice.end()); + if (idx == finish && comparator.compare(starts[idx], slice.start()) < 0) + return rangeTombstoneWithNewStart(idx--, slice.start()); + return rangeTombstone(idx++); + } + }; } - @Override public boolean equals(Object o) { @@ -525,51 +504,24 @@ public class RangeTombstoneList implements Iterable, IMeasurable /* * Inserts a new element starting at index i. This method assumes that: - * ends[i-1] <= start <= ends[i] + * ends[i-1] < start < ends[i] + * (note that we cannot have start == end since both will at least have a different bound "kind") * * A RangeTombstoneList is a list of range [s_0, e_0]...[s_n, e_n] such that: * - s_i <= e_i - * - e_i <= s_i+1 - * - if s_i == e_i and e_i == s_i+1 then s_i+1 < e_i+1 - * Basically, range are non overlapping except for their bound and in order. And while - * we allow ranges with the same value for the start and end, we don't allow repeating - * such range (so we can't have [0, 0][0, 0] even though it would respect the first 2 - * conditions). - * + * - e_i < s_i+1 + * Basically, range are non overlapping and in order. */ - private void insertFrom(int i, Composite start, Composite end, long markedAt, int delTime) + private void insertFrom(int i, Slice.Bound start, Slice.Bound end, long markedAt, int delTime) { while (i < size) { - assert i == 0 || comparator.compare(ends[i-1], start) <= 0; + assert start.isStart() && end.isEnd(); + assert i == 0 || comparator.compare(ends[i-1], start) < 0; + assert comparator.compare(start, ends[i]) < 0; - int c = comparator.compare(start, ends[i]); - assert c <= 0; - if (c == 0) - { - // If start == ends[i], then we can insert from the next one (basically the new element - // really start at the next element), except for the case where starts[i] == ends[i]. - // In this latter case, if we were to move to next element, we could end up with ...[x, x][x, x]... - if (comparator.compare(starts[i], ends[i]) == 0) - { - // The current element cover a single value which is equal to the start of the inserted - // element. If the inserted element overwrites the current one, just remove the current - // (it's included in what we insert) and proceed with the insert. - if (markedAt > markedAts[i]) - { - removeInternal(i); - continue; - } - - // Otherwise (the current singleton interval override the new one), we want to leave the - // current element and move to the next, unless start == end since that means the new element - // is in fact fully covered by the current one (so we're done) - if (comparator.compare(start, end) == 0) - return; - } - i++; - continue; - } + if (Slice.isEmpty(comparator, start, end)) + return; // Do we overwrite the current element? if (markedAt > markedAts[i]) @@ -579,26 +531,24 @@ public class RangeTombstoneList implements Iterable, IMeasurable // First deal with what might come before the newly added one. if (comparator.compare(starts[i], start) < 0) { - addInternal(i, starts[i], start, markedAts[i], delTimes[i]); - i++; - // We don't need to do the following line, but in spirit that's what we want to do - // setInternal(i, start, ends[i], markedAts, delTime]) + Slice.Bound newEnd = start.invert(); + if (!Slice.isEmpty(comparator, starts[i], newEnd)) + { + addInternal(i, starts[i], start.invert(), markedAts[i], delTimes[i]); + i++; + setInternal(i, start, ends[i], markedAts[i], delTimes[i]); + } } // now, start <= starts[i] - // Does the new element stops before/at the current one, + // Does the new element stops before the current one, int endCmp = comparator.compare(end, starts[i]); - if (endCmp <= 0) + if (endCmp < 0) { - // Here start <= starts[i] and end <= starts[i] - // This means the current element is before the current one. However, one special - // case is if end == starts[i] and starts[i] == ends[i]. In that case, - // the new element entirely overwrite the current one and we can just overwrite - if (endCmp == 0 && comparator.compare(starts[i], ends[i]) == 0) - setInternal(i, start, end, markedAt, delTime); - else - addInternal(i, start, end, markedAt, delTime); + // Here start <= starts[i] and end < starts[i] + // This means the current element is before the current one. + addInternal(i, start, end, markedAt, delTime); return; } @@ -617,20 +567,29 @@ public class RangeTombstoneList implements Iterable, IMeasurable return; } - setInternal(i, start, ends[i], markedAt, delTime); - if (cmp == 0) + // Otherwise, the new element overwite until the min(end, next start) + if (comparator.compare(end, starts[i+1]) < 0) + { + setInternal(i, start, end, markedAt, delTime); + // We have fully handled the new element so we're done return; + } - start = ends[i]; + setInternal(i, start, starts[i+1].invert(), markedAt, delTime); + start = starts[i+1]; i++; } else { - // We don't ovewrite fully. Insert the new interval, and then update the now next + // We don't overwrite fully. Insert the new interval, and then update the now next // one to reflect the not overwritten parts. We're then done. addInternal(i, start, end, markedAt, delTime); i++; - setInternal(i, end, ends[i], markedAts[i], delTimes[i]); + Slice.Bound newStart = end.invert(); + if (!Slice.isEmpty(comparator, newStart, ends[i])) + { + setInternal(i, newStart, ends[i], markedAts[i], delTimes[i]); + } return; } } @@ -644,13 +603,17 @@ public class RangeTombstoneList implements Iterable, IMeasurable // If we stop before the start of the current element, just insert the new // interval and we're done; otherwise insert until the beginning of the // current element - if (comparator.compare(end, starts[i]) <= 0) + if (comparator.compare(end, starts[i]) < 0) { addInternal(i, start, end, markedAt, delTime); return; } - addInternal(i, start, starts[i], markedAt, delTime); - i++; + Slice.Bound newEnd = starts[i].invert(); + if (!Slice.isEmpty(comparator, start, newEnd)) + { + addInternal(i, start, newEnd, markedAt, delTime); + i++; + } } // After that, we're overwritten on the current element but might have @@ -660,7 +623,7 @@ public class RangeTombstoneList implements Iterable, IMeasurable if (comparator.compare(end, ends[i]) <= 0) return; - start = ends[i]; + start = ends[i].invert(); i++; } } @@ -677,7 +640,7 @@ public class RangeTombstoneList implements Iterable, IMeasurable /* * Adds the new tombstone at index i, growing and/or moving elements to make room for it. */ - private void addInternal(int i, Composite start, Composite end, long markedAt, int delTime) + private void addInternal(int i, Slice.Bound start, Slice.Bound end, long markedAt, int delTime) { assert i >= 0; @@ -730,12 +693,12 @@ public class RangeTombstoneList implements Iterable, IMeasurable delTimes = grow(delTimes, size, newLength, i); } - private static Composite[] grow(Composite[] a, int size, int newLength, int i) + private static Slice.Bound[] grow(Slice.Bound[] a, int size, int newLength, int i) { if (i < 0 || i >= size) return Arrays.copyOf(a, newLength); - Composite[] newA = new Composite[newLength]; + Slice.Bound[] newA = new Slice.Bound[newLength]; System.arraycopy(a, 0, newA, 0, i); System.arraycopy(a, i, newA, i+1, size - i); return newA; @@ -780,7 +743,7 @@ public class RangeTombstoneList implements Iterable, IMeasurable starts[i] = null; } - private void setInternal(int i, Composite start, Composite end, long markedAt, int delTime) + private void setInternal(int i, Slice.Bound start, Slice.Bound end, long markedAt, int delTime) { if (starts[i] != null) boundaryHeapSize -= starts[i].unsharedHeapSize() + ends[i].unsharedHeapSize(); @@ -802,82 +765,91 @@ public class RangeTombstoneList implements Iterable, IMeasurable + ObjectSizes.sizeOfArray(delTimes); } + // TODO: This should be moved someplace else as it shouldn't be used directly: some ranges might become + // complex deletion times. We'll also only need this for backward compatibility, this isn't used otherwise. public static class Serializer implements IVersionedSerializer { - private final CType type; + private final LegacyLayout layout; - public Serializer(CType type) + public Serializer(LegacyLayout layout) { - this.type = type; + this.layout = layout; } public void serialize(RangeTombstoneList tombstones, DataOutputPlus out, int version) throws IOException { - if (tombstones == null) - { - out.writeInt(0); - return; - } + // TODO + throw new UnsupportedOperationException(); + //if (tombstones == null) + //{ + // out.writeInt(0); + // return; + //} - out.writeInt(tombstones.size); - for (int i = 0; i < tombstones.size; i++) - { - type.serializer().serialize(tombstones.starts[i], out); - type.serializer().serialize(tombstones.ends[i], out); - out.writeInt(tombstones.delTimes[i]); - out.writeLong(tombstones.markedAts[i]); - } + //out.writeInt(tombstones.size); + //for (int i = 0; i < tombstones.size; i++) + //{ + // layout.serializer().serialize(tombstones.starts[i], out); + // layout.serializer().serialize(tombstones.ends[i], out); + // out.writeInt(tombstones.delTimes[i]); + // out.writeLong(tombstones.markedAts[i]); + //} } public RangeTombstoneList deserialize(DataInput in, int version) throws IOException { - int size = in.readInt(); - if (size == 0) - return null; + // TODO + throw new UnsupportedOperationException(); - RangeTombstoneList tombstones = new RangeTombstoneList(type, size); + //int size = in.readInt(); + //if (size == 0) + // return null; - for (int i = 0; i < size; i++) - { - Composite start = type.serializer().deserialize(in); - Composite end = type.serializer().deserialize(in); - int delTime = in.readInt(); - long markedAt = in.readLong(); + //RangeTombstoneList tombstones = new RangeTombstoneList(layout, size); - if (version >= MessagingService.VERSION_20) - { - tombstones.setInternal(i, start, end, markedAt, delTime); - } - else - { - /* - * The old implementation used to have range sorted by left value, but with potentially - * overlapping range. So we need to use the "slow" path. - */ - tombstones.add(start, end, markedAt, delTime); - } - } + //for (int i = 0; i < size; i++) + //{ + // Slice.Bound start = layout.serializer().deserialize(in); + // Slice.Bound end = layout.serializer().deserialize(in); + // int delTime = in.readInt(); + // long markedAt = in.readLong(); - // The "slow" path take care of updating the size, but not the fast one - if (version >= MessagingService.VERSION_20) - tombstones.size = size; - return tombstones; + // if (version >= MessagingService.VERSION_20) + // { + // tombstones.setInternal(i, start, end, markedAt, delTime); + // } + // else + // { + // /* + // * The old implementation used to have range sorted by left value, but with potentially + // * overlapping range. So we need to use the "slow" path. + // */ + // tombstones.add(start, end, markedAt, delTime); + // } + //} + + //// The "slow" path take care of updating the size, but not the fast one + //if (version >= MessagingService.VERSION_20) + // tombstones.size = size; + //return tombstones; } public long serializedSize(RangeTombstoneList tombstones, TypeSizes typeSizes, int version) { - if (tombstones == null) - return typeSizes.sizeof(0); + // TODO + throw new UnsupportedOperationException(); + //if (tombstones == null) + // return typeSizes.sizeof(0); - long size = typeSizes.sizeof(tombstones.size); - for (int i = 0; i < tombstones.size; i++) - { - size += type.serializer().serializedSize(tombstones.starts[i], typeSizes); - size += type.serializer().serializedSize(tombstones.ends[i], typeSizes); - size += typeSizes.sizeof(tombstones.delTimes[i]); - size += typeSizes.sizeof(tombstones.markedAts[i]); - } - return size; + //long size = typeSizes.sizeof(tombstones.size); + //for (int i = 0; i < tombstones.size; i++) + //{ + // size += type.serializer().serializedSize(tombstones.starts[i], typeSizes); + // size += type.serializer().serializedSize(tombstones.ends[i], typeSizes); + // size += typeSizes.sizeof(tombstones.delTimes[i]); + // size += typeSizes.sizeof(tombstones.markedAts[i]); + //} + //return size; } public long serializedSize(RangeTombstoneList tombstones, int version) @@ -885,56 +857,4 @@ public class RangeTombstoneList implements Iterable, IMeasurable return serializedSize(tombstones, TypeSizes.NATIVE, version); } } - - /** - * This object allow testing whether a given column (name/timestamp) is deleted - * or not by this RangeTombstoneList, assuming that the column given to this - * object are passed in (comparator) sorted order. - * - * This is more efficient that calling RangeTombstoneList.isDeleted() repeatedly - * in that case since we're able to take the sorted nature of the RangeTombstoneList - * into account. - */ - public class InOrderTester - { - private int idx; - - public boolean isDeleted(Cell cell) - { - CellName name = cell.name(); - long timestamp = cell.timestamp(); - - while (idx < size) - { - int cmp = comparator.compare(name, starts[idx]); - - if (cmp < 0) - { - return false; - } - else if (cmp == 0) - { - // No matter what the counter cell's timestamp is, a tombstone always takes precedence. See CASSANDRA-7346. - if (cell instanceof CounterCell) - return true; - - // As for searchInternal, we need to check the previous end - if (idx > 0 && comparator.compare(name, ends[idx-1]) == 0 && markedAts[idx-1] > markedAts[idx]) - return markedAts[idx-1] >= timestamp; - else - return markedAts[idx] >= timestamp; - } - else - { - if (comparator.compare(name, ends[idx]) <= 0) - return markedAts[idx] >= timestamp || cell instanceof CounterCell; - else - idx++; - } - } - - return false; - } - } - } diff --git a/src/java/org/apache/cassandra/db/ReadCommand.java b/src/java/org/apache/cassandra/db/ReadCommand.java index dedff6faaf..bad096f21c 100644 --- a/src/java/org/apache/cassandra/db/ReadCommand.java +++ b/src/java/org/apache/cassandra/db/ReadCommand.java @@ -19,158 +19,500 @@ package org.apache.cassandra.db; import java.io.DataInput; import java.io.IOException; -import java.nio.ByteBuffer; +import com.google.common.collect.Iterables; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import org.apache.cassandra.config.CFMetaData; +import org.apache.cassandra.config.ColumnDefinition; import org.apache.cassandra.config.DatabaseDescriptor; -import org.apache.cassandra.db.filter.IDiskAtomFilter; -import org.apache.cassandra.db.filter.NamesQueryFilter; -import org.apache.cassandra.db.filter.SliceQueryFilter; +import org.apache.cassandra.db.index.SecondaryIndexSearcher; +import org.apache.cassandra.db.filter.*; +import org.apache.cassandra.db.rows.*; +import org.apache.cassandra.db.partitions.*; import org.apache.cassandra.io.IVersionedSerializer; import org.apache.cassandra.io.util.DataOutputPlus; +import org.apache.cassandra.metrics.ColumnFamilyMetrics; import org.apache.cassandra.net.MessageOut; import org.apache.cassandra.net.MessagingService; -import org.apache.cassandra.service.IReadCommand; -import org.apache.cassandra.service.RowDataResolver; -import org.apache.cassandra.service.pager.Pageable; +import org.apache.cassandra.service.ClientWarn; +import org.apache.cassandra.tracing.Tracing; -public abstract class ReadCommand implements IReadCommand, Pageable +/** + * General interface for storage-engine read commands (common to both range and + * single partition commands). + *

+ * This contains all the informations needed to do a local read. + */ +public abstract class ReadCommand implements ReadQuery { - public enum Type + protected static final Logger logger = LoggerFactory.getLogger(ReadCommand.class); + + public static final IVersionedSerializer serializer = new Serializer(); + + private final Kind kind; + private final CFMetaData metadata; + private final int nowInSec; + + private final ColumnFilter columnFilter; + private final RowFilter rowFilter; + private final DataLimits limits; + + private boolean isDigestQuery; + private final boolean isForThrift; + + protected static abstract class SelectionDeserializer { - GET_BY_NAMES((byte)1), - GET_SLICES((byte)2); + public abstract ReadCommand deserialize(DataInput in, int version, boolean isDigest, boolean isForThrift, CFMetaData metadata, int nowInSec, ColumnFilter columnFilter, RowFilter rowFilter, DataLimits limits) throws IOException; + } - public final byte serializedValue; + protected enum Kind + { + SINGLE_PARTITION (SinglePartitionReadCommand.selectionDeserializer), + PARTITION_RANGE (PartitionRangeReadCommand.selectionDeserializer); - private Type(byte b) + private SelectionDeserializer selectionDeserializer; + + private Kind(SelectionDeserializer selectionDeserializer) { - this.serializedValue = b; - } - - public static Type fromSerializedValue(byte b) - { - return b == 1 ? GET_BY_NAMES : GET_SLICES; + this.selectionDeserializer = selectionDeserializer; } } - public static final ReadCommandSerializer serializer = new ReadCommandSerializer(); - - public MessageOut createMessage() + protected ReadCommand(Kind kind, + boolean isDigestQuery, + boolean isForThrift, + CFMetaData metadata, + int nowInSec, + ColumnFilter columnFilter, + RowFilter rowFilter, + DataLimits limits) { - return new MessageOut<>(MessagingService.Verb.READ, this, serializer); + this.kind = kind; + this.isDigestQuery = isDigestQuery; + this.isForThrift = isForThrift; + this.metadata = metadata; + this.nowInSec = nowInSec; + this.columnFilter = columnFilter; + this.rowFilter = rowFilter; + this.limits = limits; } - public final String ksName; - public final String cfName; - public final ByteBuffer key; - public final long timestamp; - private boolean isDigestQuery = false; - protected final Type commandType; + protected abstract void serializeSelection(DataOutputPlus out, int version) throws IOException; + protected abstract long selectionSerializedSize(int version); - protected ReadCommand(String ksName, ByteBuffer key, String cfName, long timestamp, Type cmdType) + /** + * The metadata for the table queried. + * + * @return the metadata for the table queried. + */ + public CFMetaData metadata() { - this.ksName = ksName; - this.key = key; - this.cfName = cfName; - this.timestamp = timestamp; - this.commandType = cmdType; + return metadata; } - public static ReadCommand create(String ksName, ByteBuffer key, String cfName, long timestamp, IDiskAtomFilter filter) + /** + * The time in seconds to use as "now" for this query. + *

+ * We use the same time as "now" for the whole query to avoid considering different + * values as expired during the query, which would be buggy (would throw of counting amongst other + * things). + * + * @return the time (in seconds) to use as "now". + */ + public int nowInSec() { - if (filter instanceof SliceQueryFilter) - return new SliceFromReadCommand(ksName, key, cfName, timestamp, (SliceQueryFilter)filter); - else - return new SliceByNamesReadCommand(ksName, key, cfName, timestamp, (NamesQueryFilter)filter); + return nowInSec; } + /** + * The configured timeout for this command. + * + * @return the configured timeout for this command. + */ + public abstract long getTimeout(); + + /** + * A filter on which (non-PK) columns must be returned by the query. + * + * @return which columns must be fetched by this query. + */ + public ColumnFilter columnFilter() + { + return columnFilter; + } + + /** + * Filters/Resrictions on CQL rows. + *

+ * This contains the restrictions that are not directly handled by the + * {@code ClusteringIndexFilter}. More specifically, this includes any non-PK column + * restrictions and can include some PK columns restrictions when those can't be + * satisfied entirely by the clustering index filter (because not all clustering columns + * have been restricted for instance). If there is 2ndary indexes on the table, + * one of this restriction might be handled by a 2ndary index. + * + * @return the filter holding the expression that rows must satisfy. + */ + public RowFilter rowFilter() + { + return rowFilter; + } + + /** + * The limits set on this query. + * + * @return the limits set on this query. + */ + public DataLimits limits() + { + return limits; + } + + /** + * Whether this query is a digest one or not. + * + * @return Whether this query is a digest query. + */ public boolean isDigestQuery() { return isDigestQuery; } + /** + * Sets whether this command should be a digest one or not. + * + * @param isDigestQuery whether the command should be set as a digest one or not. + * @return this read command. + */ public ReadCommand setIsDigestQuery(boolean isDigestQuery) { this.isDigestQuery = isDigestQuery; return this; } - public String getColumnFamilyName() + /** + * Whether this query is for thrift or not. + * + * @return whether this query is for thrift. + */ + public boolean isForThrift() { - return cfName; + return isForThrift; } + /** + * The clustering index filter this command to use for the provided key. + *

+ * Note that that method should only be called on a key actually queried by this command + * and in practice, this will almost always return the same filter, but for the sake of + * paging, the filter on the first key of a range command might be slightly different. + * + * @param key a partition key queried by this command. + * + * @return the {@code ClusteringIndexFilter} to use for the partition of key {@code key}. + */ + public abstract ClusteringIndexFilter clusteringIndexFilter(DecoratedKey key); + + /** + * Returns a copy of this command. + * + * @return a copy of this command. + */ public abstract ReadCommand copy(); - public abstract Row getRow(Keyspace keyspace); + /** + * Whether the provided row, identified by its primary key components, is selected by + * this read command. + * + * @param partitionKey the partition key for the row to test. + * @param clustering the clustering for the row to test. + * + * @return whether the row of partition key {@code partitionKey} and clustering + * {@code clustering} is selected by this command. + */ + public abstract boolean selects(DecoratedKey partitionKey, Clustering clustering); - public abstract IDiskAtomFilter filter(); + protected abstract UnfilteredPartitionIterator queryStorage(ColumnFamilyStore cfs, ReadOrderGroup orderGroup); - public String getKeyspace() + public ReadResponse createResponse(UnfilteredPartitionIterator iterator) { - return ksName; + return isDigestQuery() + ? ReadResponse.createDigestResponse(iterator) + : ReadResponse.createDataResponse(iterator); } - // maybeGenerateRetryCommand is used to generate a retry for short reads - public ReadCommand maybeGenerateRetryCommand(RowDataResolver resolver, Row row) + protected SecondaryIndexSearcher getIndexSearcher(ColumnFamilyStore cfs) { - return null; + return cfs.indexManager.getBestIndexSearcherFor(this); } - // maybeTrim removes columns from a response that is too long - public void maybeTrim(Row row) + /** + * Executes this command on the local host. + * + * @param cfs the store for the table queried by this command. + * + * @return an iterator over the result of executing this command locally. + */ + @SuppressWarnings("resource") // The result iterator is closed upon exceptions (we know it's fine to potentially not close the intermediary + // iterators created inside the try as long as we do close the original resultIterator), or by closing the result. + public UnfilteredPartitionIterator executeLocally(ReadOrderGroup orderGroup) { - // noop - } + long startTimeNanos = System.nanoTime(); - public long getTimeout() - { - return DatabaseDescriptor.getReadRpcTimeout(); - } -} + ColumnFamilyStore cfs = Keyspace.openAndGetStore(metadata()); + SecondaryIndexSearcher searcher = getIndexSearcher(cfs); + UnfilteredPartitionIterator resultIterator = searcher == null + ? queryStorage(cfs, orderGroup) + : searcher.search(this, orderGroup); -class ReadCommandSerializer implements IVersionedSerializer -{ - public void serialize(ReadCommand command, DataOutputPlus out, int version) throws IOException - { - out.writeByte(command.commandType.serializedValue); - switch (command.commandType) + try { - case GET_BY_NAMES: - SliceByNamesReadCommand.serializer.serialize(command, out, version); - break; - case GET_SLICES: - SliceFromReadCommand.serializer.serialize(command, out, version); - break; - default: - throw new AssertionError(); + resultIterator = UnfilteredPartitionIterators.convertExpiredCellsToTombstones(resultIterator, nowInSec); + resultIterator = withMetricsRecording(withoutExpiredTombstones(resultIterator, cfs), cfs.metric, startTimeNanos); + + // TODO: we should push the dropping of columns down the layers because + // 1) it'll be more efficient + // 2) it could help us solve #6276 + // But there is not reason not to do this as a followup so keeping it here for now (we'll have + // to be wary of cached row if we move this down the layers) + if (!metadata().getDroppedColumns().isEmpty()) + resultIterator = UnfilteredPartitionIterators.removeDroppedColumns(resultIterator, metadata().getDroppedColumns()); + + // If we've used a 2ndary index, we know the result already satisfy the primary expression used, so + // no point in checking it again. + RowFilter updatedFilter = searcher == null + ? rowFilter() + : rowFilter().without(searcher.primaryClause(this)); + + // TODO: We'll currently do filtering by the rowFilter here because it's convenient. However, + // we'll probably want to optimize by pushing it down the layer (like for dropped columns) as it + // would be more efficient (the sooner we discard stuff we know we don't care, the less useless + // processing we do on it). + return limits().filter(rowFilter().filter(resultIterator, nowInSec()), nowInSec()); + } + catch (RuntimeException | Error e) + { + resultIterator.close(); + throw e; } } - public ReadCommand deserialize(DataInput in, int version) throws IOException + protected abstract void recordLatency(ColumnFamilyMetrics metric, long latencyNanos); + + public PartitionIterator executeInternal(ReadOrderGroup orderGroup) { - ReadCommand.Type msgType = ReadCommand.Type.fromSerializedValue(in.readByte()); - switch (msgType) - { - case GET_BY_NAMES: - return SliceByNamesReadCommand.serializer.deserialize(in, version); - case GET_SLICES: - return SliceFromReadCommand.serializer.deserialize(in, version); - default: - throw new AssertionError(); - } + return UnfilteredPartitionIterators.filter(executeLocally(orderGroup), nowInSec()); } - public long serializedSize(ReadCommand command, int version) + public ReadOrderGroup startOrderGroup() { - switch (command.commandType) + return ReadOrderGroup.forCommand(this); + } + + /** + * Wraps the provided iterator so that metrics on what is scanned by the command are recorded. + * This also log warning/trow TombstoneOverwhelmingException if appropriate. + */ + private UnfilteredPartitionIterator withMetricsRecording(UnfilteredPartitionIterator iter, final ColumnFamilyMetrics metric, final long startTimeNanos) + { + return new WrappingUnfilteredPartitionIterator(iter) { - case GET_BY_NAMES: - return 1 + SliceByNamesReadCommand.serializer.serializedSize(command, version); - case GET_SLICES: - return 1 + SliceFromReadCommand.serializer.serializedSize(command, version); - default: - throw new AssertionError(); + private final int failureThreshold = DatabaseDescriptor.getTombstoneFailureThreshold(); + private final int warningThreshold = DatabaseDescriptor.getTombstoneWarnThreshold(); + + private final boolean respectTombstoneThresholds = !ReadCommand.this.metadata().ksName.equals(SystemKeyspace.NAME); + + private int liveRows = 0; + private int tombstones = 0; + + private DecoratedKey currentKey; + + @Override + public UnfilteredRowIterator computeNext(UnfilteredRowIterator iter) + { + currentKey = iter.partitionKey(); + + return new WrappingUnfilteredRowIterator(iter) + { + public Unfiltered next() + { + Unfiltered unfiltered = super.next(); + if (unfiltered.kind() == Unfiltered.Kind.ROW) + { + Row row = (Row) unfiltered; + if (row.hasLiveData(ReadCommand.this.nowInSec())) + ++liveRows; + for (Cell cell : row) + if (!cell.isLive(ReadCommand.this.nowInSec())) + countTombstone(row.clustering()); + } + else + { + countTombstone(unfiltered.clustering()); + } + + return unfiltered; + } + + private void countTombstone(ClusteringPrefix clustering) + { + ++tombstones; + if (tombstones > failureThreshold && respectTombstoneThresholds) + { + String query = ReadCommand.this.toCQLString(); + Tracing.trace("Scanned over {} tombstones for query {}; query aborted (see tombstone_failure_threshold)", failureThreshold, query); + throw new TombstoneOverwhelmingException(tombstones, query, ReadCommand.this.metadata(), currentKey, clustering); + } + } + }; + } + + @Override + public void close() + { + try + { + super.close(); + } + finally + { + recordLatency(metric, System.nanoTime() - startTimeNanos); + + metric.tombstoneScannedHistogram.update(tombstones); + metric.liveScannedHistogram.update(liveRows); + + boolean warnTombstones = tombstones > warningThreshold && respectTombstoneThresholds; + if (warnTombstones) + { + String msg = String.format("Read %d live rows and %d tombstone cells for query %1.512s (see tombstone_warn_threshold)", liveRows, tombstones, ReadCommand.this.toCQLString()); + ClientWarn.warn(msg); + logger.warn(msg); + } + + Tracing.trace("Read {} live and {} tombstone cells{}", new Object[]{ liveRows, tombstones, (warnTombstones ? " (see tombstone_warn_threshold)" : "") }); + } + } + }; + } + + /** + * Creates a message for this command. + */ + public MessageOut createMessage() + { + // TODO: we should use different verbs for old message (RANGE_SLICE, PAGED_RANGE) + return new MessageOut<>(MessagingService.Verb.READ, this, serializer); + } + + protected abstract void appendCQLWhereClause(StringBuilder sb); + + // Skip expired tombstones. We do this because it's safe to do (post-merge of the memtable and sstable at least), it + // can save us some bandwith, and avoid making us throw a TombstoneOverwhelmingException for expired tombstones (which + // are to some extend an artefact of compaction lagging behind and hence counting them is somewhat unintuitive). + protected UnfilteredPartitionIterator withoutExpiredTombstones(UnfilteredPartitionIterator iterator, ColumnFamilyStore cfs) + { + return new TombstonePurgingPartitionIterator(iterator, cfs.gcBefore(nowInSec())) + { + protected long getMaxPurgeableTimestamp() + { + return Long.MAX_VALUE; + } + }; + } + + /** + * Recreate the CQL string corresponding to this query. + *

+ * Note that in general the returned string will not be exactly the original user string, first + * because there isn't always a single syntax for a given query, but also because we don't have + * all the information needed (we know the non-PK columns queried but not the PK ones as internally + * we query them all). So this shouldn't be relied too strongly, but this should be good enough for + * debugging purpose which is what this is for. + */ + public String toCQLString() + { + StringBuilder sb = new StringBuilder(); + sb.append("SELECT ").append(columnFilter()); + sb.append(" FROM ").append(metadata().ksName).append(".").append(metadata.cfName); + appendCQLWhereClause(sb); + + if (limits() != DataLimits.NONE) + sb.append(" ").append(limits()); + return sb.toString(); + } + + private static class Serializer implements IVersionedSerializer + { + private static int digestFlag(boolean isDigest) + { + return isDigest ? 0x01 : 0; + } + + private static boolean isDigest(int flags) + { + return (flags & 0x01) != 0; + } + + private static int thriftFlag(boolean isForThrift) + { + return isForThrift ? 0x02 : 0; + } + + private static boolean isForThrift(int flags) + { + return (flags & 0x02) != 0; + } + + public void serialize(ReadCommand command, DataOutputPlus out, int version) throws IOException + { + if (version < MessagingService.VERSION_30) + throw new UnsupportedOperationException(); + + out.writeByte(command.kind.ordinal()); + out.writeByte(digestFlag(command.isDigestQuery()) | thriftFlag(command.isForThrift())); + CFMetaData.serializer.serialize(command.metadata(), out, version); + out.writeInt(command.nowInSec()); + ColumnFilter.serializer.serialize(command.columnFilter(), out, version); + RowFilter.serializer.serialize(command.rowFilter(), out, version); + DataLimits.serializer.serialize(command.limits(), out, version); + + command.serializeSelection(out, version); + } + + public ReadCommand deserialize(DataInput in, int version) throws IOException + { + if (version < MessagingService.VERSION_30) + throw new UnsupportedOperationException(); + + Kind kind = Kind.values()[in.readByte()]; + int flags = in.readByte(); + boolean isDigest = isDigest(flags); + boolean isForThrift = isForThrift(flags); + CFMetaData metadata = CFMetaData.serializer.deserialize(in, version); + int nowInSec = in.readInt(); + ColumnFilter columnFilter = ColumnFilter.serializer.deserialize(in, version, metadata); + RowFilter rowFilter = RowFilter.serializer.deserialize(in, version, metadata); + DataLimits limits = DataLimits.serializer.deserialize(in, version); + + return kind.selectionDeserializer.deserialize(in, version, isDigest, isForThrift, metadata, nowInSec, columnFilter, rowFilter, limits); + } + + public long serializedSize(ReadCommand command, int version) + { + if (version < MessagingService.VERSION_30) + throw new UnsupportedOperationException(); + + TypeSizes sizes = TypeSizes.NATIVE; + + return 2 // kind + flags + + CFMetaData.serializer.serializedSize(command.metadata(), version, sizes) + + sizes.sizeof(command.nowInSec()) + + ColumnFilter.serializer.serializedSize(command.columnFilter(), version, sizes) + + RowFilter.serializer.serializedSize(command.rowFilter(), version) + + DataLimits.serializer.serializedSize(command.limits(), version) + + command.selectionSerializedSize(version); } } } diff --git a/src/java/org/apache/cassandra/db/ReadVerbHandler.java b/src/java/org/apache/cassandra/db/ReadCommandVerbHandler.java similarity index 68% rename from src/java/org/apache/cassandra/db/ReadVerbHandler.java rename to src/java/org/apache/cassandra/db/ReadCommandVerbHandler.java index 8c167ed73e..f85d4068eb 100644 --- a/src/java/org/apache/cassandra/db/ReadVerbHandler.java +++ b/src/java/org/apache/cassandra/db/ReadCommandVerbHandler.java @@ -17,6 +17,8 @@ */ package org.apache.cassandra.db; +import org.apache.cassandra.config.CFMetaData; +import org.apache.cassandra.db.partitions.UnfilteredPartitionIterator; import org.apache.cassandra.net.IVerbHandler; import org.apache.cassandra.net.MessageIn; import org.apache.cassandra.net.MessageOut; @@ -24,7 +26,7 @@ import org.apache.cassandra.net.MessagingService; import org.apache.cassandra.service.StorageService; import org.apache.cassandra.tracing.Tracing; -public class ReadVerbHandler implements IVerbHandler +public class ReadCommandVerbHandler implements IVerbHandler { public void doVerb(MessageIn message, int id) { @@ -34,25 +36,15 @@ public class ReadVerbHandler implements IVerbHandler } ReadCommand command = message.payload; - Keyspace keyspace = Keyspace.open(command.ksName); - Row row = command.getRow(keyspace); + ReadResponse response; + try (ReadOrderGroup opGroup = command.startOrderGroup(); UnfilteredPartitionIterator iterator = command.executeLocally(opGroup)) + { + response = command.createResponse(iterator); + } + + MessageOut reply = new MessageOut<>(MessagingService.Verb.REQUEST_RESPONSE, response, ReadResponse.serializer); - MessageOut reply = new MessageOut(MessagingService.Verb.REQUEST_RESPONSE, - getResponse(command, row), - ReadResponse.serializer); Tracing.trace("Enqueuing response to {}", message.from); MessagingService.instance().sendReply(reply, id, message.from); } - - public static ReadResponse getResponse(ReadCommand command, Row row) - { - if (command.isDigestQuery()) - { - return new ReadResponse(ColumnFamily.digest(row.cf)); - } - else - { - return new ReadResponse(row); - } - } } diff --git a/src/java/org/apache/cassandra/db/ReadOrderGroup.java b/src/java/org/apache/cassandra/db/ReadOrderGroup.java new file mode 100644 index 0000000000..0a5bee8c64 --- /dev/null +++ b/src/java/org/apache/cassandra/db/ReadOrderGroup.java @@ -0,0 +1,133 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.cassandra.db; + +import org.apache.cassandra.db.index.*; +import org.apache.cassandra.utils.concurrent.OpOrder; + +public class ReadOrderGroup implements AutoCloseable +{ + // For every reads + private final OpOrder.Group baseOp; + + // For index reads + private final OpOrder.Group indexOp; + private final OpOrder.Group writeOp; + + private ReadOrderGroup(OpOrder.Group baseOp, OpOrder.Group indexOp, OpOrder.Group writeOp) + { + this.baseOp = baseOp; + this.indexOp = indexOp; + this.writeOp = writeOp; + } + + public OpOrder.Group baseReadOpOrderGroup() + { + return baseOp; + } + + public OpOrder.Group indexReadOpOrderGroup() + { + return indexOp; + } + + public OpOrder.Group writeOpOrderGroup() + { + return writeOp; + } + + public static ReadOrderGroup emptyGroup() + { + return new ReadOrderGroup(null, null, null); + } + + public static ReadOrderGroup forCommand(ReadCommand command) + { + ColumnFamilyStore baseCfs = Keyspace.openAndGetStore(command.metadata()); + ColumnFamilyStore indexCfs = maybeGetIndexCfs(baseCfs, command); + + if (indexCfs == null) + { + return new ReadOrderGroup(baseCfs.readOrdering.start(), null, null); + } + else + { + OpOrder.Group baseOp = null, indexOp = null, writeOp; + // OpOrder.start() shouldn't fail, but better safe than sorry. + try + { + baseOp = baseCfs.readOrdering.start(); + indexOp = indexCfs.readOrdering.start(); + // TODO: this should perhaps not open and maintain a writeOp for the full duration, but instead only *try* to delete stale entries, without blocking if there's no room + // as it stands, we open a writeOp and keep it open for the duration to ensure that should this CF get flushed to make room we don't block the reclamation of any room being made + writeOp = baseCfs.keyspace.writeOrder.start(); + return new ReadOrderGroup(baseOp, indexOp, writeOp); + } + catch (RuntimeException e) + { + // Note that must have writeOp == null since ReadOrderGroup ctor can't fail + try + { + if (baseOp != null) + baseOp.close(); + } + finally + { + if (indexOp != null) + indexOp.close(); + } + throw e; + } + } + } + + private static ColumnFamilyStore maybeGetIndexCfs(ColumnFamilyStore baseCfs, ReadCommand command) + { + SecondaryIndexSearcher searcher = command.getIndexSearcher(baseCfs); + if (searcher == null) + return null; + + SecondaryIndex index = searcher.highestSelectivityIndex(command.rowFilter()); + return index == null || !(index instanceof AbstractSimplePerColumnSecondaryIndex) + ? null + : ((AbstractSimplePerColumnSecondaryIndex)index).getIndexCfs(); + } + + public void close() + { + try + { + if (baseOp != null) + baseOp.close(); + } + finally + { + if (indexOp != null) + { + try + { + indexOp.close(); + } + finally + { + writeOp.close(); + } + } + } + } +} diff --git a/src/java/org/apache/cassandra/db/ReadQuery.java b/src/java/org/apache/cassandra/db/ReadQuery.java new file mode 100644 index 0000000000..3ad0f829e1 --- /dev/null +++ b/src/java/org/apache/cassandra/db/ReadQuery.java @@ -0,0 +1,118 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.cassandra.db; + +import org.apache.cassandra.db.filter.DataLimits; +import org.apache.cassandra.db.partitions.*; +import org.apache.cassandra.exceptions.RequestExecutionException; +import org.apache.cassandra.service.ClientState; +import org.apache.cassandra.service.pager.QueryPager; +import org.apache.cassandra.service.pager.PagingState; + +/** + * Generic abstraction for read queries. + *

+ * The main implementation of this is {@link ReadCommand} but we have this interface because + * {@link SinglePartitionReadCommand.Group} is also consider as a "read query" but is not a + * {@code ReadCommand}. + */ +public interface ReadQuery +{ + public static final ReadQuery EMPTY = new ReadQuery() + { + public ReadOrderGroup startOrderGroup() + { + return ReadOrderGroup.emptyGroup(); + } + + public PartitionIterator execute(ConsistencyLevel consistency, ClientState clientState) throws RequestExecutionException + { + return PartitionIterators.EMPTY; + } + + public PartitionIterator executeInternal(ReadOrderGroup orderGroup) + { + return PartitionIterators.EMPTY; + } + + public DataLimits limits() + { + // What we return here doesn't matter much in practice. However, returning DataLimits.NONE means + // "no particular limit", which makes SelectStatement.execute() take the slightly more complex "paging" + // path. Not a big deal but it's easy enough to return a limit of 0 rows which avoids this. + return DataLimits.cqlLimits(0); + } + + public QueryPager getPager(PagingState state) + { + return QueryPager.EMPTY; + } + + public QueryPager getLocalPager() + { + return QueryPager.EMPTY; + } + }; + + /** + * Starts a new read operation. + *

+ * This must be called before {@link executeInternal} and passed to it to protect the read. + * The returned object must be closed on all path and it is thus strongly advised to + * use it in a try-with-ressource construction. + * + * @return a newly started order group for this {@code ReadQuery}. + */ + public ReadOrderGroup startOrderGroup(); + + /** + * Executes the query at the provided consistency level. + * + * @param consistency the consistency level to achieve for the query. + * @param clientState the {@code ClientState} for the query. In practice, this can be null unless + * {@code consistency} is a serial consistency. + * + * @return the result of the query. + */ + public PartitionIterator execute(ConsistencyLevel consistency, ClientState clientState) throws RequestExecutionException; + + /** + * Execute the query for internal queries (that is, it basically executes the query locally). + * + * @param orderGroup the {@code ReadOrderGroup} protecting the read. + * @return the result of the query. + */ + public PartitionIterator executeInternal(ReadOrderGroup orderGroup); + + /** + * Returns a pager for the query. + * + * @param pagingState the {@code PagingState} to start from if this is a paging continuation. This can be + * {@code null} if this is the start of paging. + * + * @return a pager for the query. + */ + public QueryPager getPager(PagingState pagingState); + + /** + * The limits for the query. + * + * @return The limits for the query. + */ + public DataLimits limits(); +} diff --git a/src/java/org/apache/cassandra/db/ReadResponse.java b/src/java/org/apache/cassandra/db/ReadResponse.java index 39022a4071..6453077c8c 100644 --- a/src/java/org/apache/cassandra/db/ReadResponse.java +++ b/src/java/org/apache/cassandra/db/ReadResponse.java @@ -19,96 +19,219 @@ package org.apache.cassandra.db; import java.io.*; import java.nio.ByteBuffer; +import java.security.MessageDigest; +import java.util.ArrayList; +import java.util.List; +import org.apache.cassandra.db.rows.*; +import org.apache.cassandra.db.partitions.*; import org.apache.cassandra.io.IVersionedSerializer; import org.apache.cassandra.io.util.DataOutputPlus; +import org.apache.cassandra.io.util.DataOutputBuffer; +import org.apache.cassandra.net.MessagingService; import org.apache.cassandra.utils.ByteBufferUtil; +import org.apache.cassandra.utils.FBUtilities; -/* - * The read response message is sent by the server when reading data - * this encapsulates the keyspacename and the row that has been read. - * The keyspace name is needed so that we can use it to create repairs. - */ -public class ReadResponse +public abstract class ReadResponse { - public static final IVersionedSerializer serializer = new ReadResponseSerializer(); + public static final IVersionedSerializer serializer = new Serializer(); + public static final IVersionedSerializer legacyRangeSliceReplySerializer = new LegacyRangeSliceReplySerializer(); - private final Row row; - private final ByteBuffer digest; - - public ReadResponse(ByteBuffer digest) + public static ReadResponse createDataResponse(UnfilteredPartitionIterator data) { - assert digest != null; - this.digest= digest; - this.row = null; + return new DataResponse(data); } - public ReadResponse(Row row) + public static ReadResponse createDigestResponse(UnfilteredPartitionIterator data) { - assert row != null; - this.row = row; - this.digest = null; + return new DigestResponse(makeDigest(data)); } - public Row row() + public abstract UnfilteredPartitionIterator makeIterator(); + public abstract ByteBuffer digest(); + public abstract boolean isDigestQuery(); + + protected static ByteBuffer makeDigest(UnfilteredPartitionIterator iterator) { - return row; + MessageDigest digest = FBUtilities.threadLocalMD5Digest(); + UnfilteredPartitionIterators.digest(iterator, digest); + return ByteBuffer.wrap(digest.digest()); } - public ByteBuffer digest() + private static class DigestResponse extends ReadResponse { - return digest; - } + private final ByteBuffer digest; - public boolean isDigestQuery() - { - return digest != null; - } -} - -class ReadResponseSerializer implements IVersionedSerializer -{ - public void serialize(ReadResponse response, DataOutputPlus out, int version) throws IOException - { - out.writeInt(response.isDigestQuery() ? response.digest().remaining() : 0); - ByteBuffer buffer = response.isDigestQuery() ? response.digest() : ByteBufferUtil.EMPTY_BYTE_BUFFER; - out.write(buffer); - out.writeBoolean(response.isDigestQuery()); - if (!response.isDigestQuery()) - Row.serializer.serialize(response.row(), out, version); - } - - public ReadResponse deserialize(DataInput in, int version) throws IOException - { - byte[] digest = null; - int digestSize = in.readInt(); - if (digestSize > 0) + private DigestResponse(ByteBuffer digest) { - digest = new byte[digestSize]; - in.readFully(digest, 0, digestSize); - } - boolean isDigest = in.readBoolean(); - assert isDigest == digestSize > 0; - - Row row = null; - if (!isDigest) - { - // This is coming from a remote host - row = Row.serializer.deserialize(in, version, ColumnSerializer.Flag.FROM_REMOTE); + assert digest.hasRemaining(); + this.digest = digest; } - return isDigest ? new ReadResponse(ByteBuffer.wrap(digest)) : new ReadResponse(row); + public UnfilteredPartitionIterator makeIterator() + { + throw new UnsupportedOperationException(); + } + + public ByteBuffer digest() + { + return digest; + } + + public boolean isDigestQuery() + { + return true; + } } - public long serializedSize(ReadResponse response, int version) + private static class DataResponse extends ReadResponse { - TypeSizes typeSizes = TypeSizes.NATIVE; - ByteBuffer buffer = response.isDigestQuery() ? response.digest() : ByteBufferUtil.EMPTY_BYTE_BUFFER; - int size = typeSizes.sizeof(buffer.remaining()); - size += buffer.remaining(); - size += typeSizes.sizeof(response.isDigestQuery()); - if (!response.isDigestQuery()) - size += Row.serializer.serializedSize(response.row(), version); - return size; + // The response, serialized in the current messaging version + private final ByteBuffer data; + private final SerializationHelper.Flag flag; + + private DataResponse(ByteBuffer data) + { + this.data = data; + this.flag = SerializationHelper.Flag.FROM_REMOTE; + } + + private DataResponse(UnfilteredPartitionIterator iter) + { + try (DataOutputBuffer buffer = new DataOutputBuffer()) + { + UnfilteredPartitionIterators.serializerForIntraNode().serialize(iter, buffer, MessagingService.current_version); + this.data = buffer.buffer(); + this.flag = SerializationHelper.Flag.LOCAL; + } + catch (IOException e) + { + // We're serializing in memory so this shouldn't happen + throw new RuntimeException(e); + } + } + + public UnfilteredPartitionIterator makeIterator() + { + try + { + DataInput in = new DataInputStream(ByteBufferUtil.inputStream(data)); + return UnfilteredPartitionIterators.serializerForIntraNode().deserialize(in, MessagingService.current_version, flag); + } + catch (IOException e) + { + // We're deserializing in memory so this shouldn't happen + throw new RuntimeException(e); + } + } + + public ByteBuffer digest() + { + try (UnfilteredPartitionIterator iterator = makeIterator()) + { + return makeDigest(iterator); + } + } + + public boolean isDigestQuery() + { + return false; + } + } + + private static class Serializer implements IVersionedSerializer + { + public void serialize(ReadResponse response, DataOutputPlus out, int version) throws IOException + { + if (version < MessagingService.VERSION_30) + { + // TODO + throw new UnsupportedOperationException(); + } + + boolean isDigest = response.isDigestQuery(); + ByteBufferUtil.writeWithShortLength(isDigest ? response.digest() : ByteBufferUtil.EMPTY_BYTE_BUFFER, out); + if (!isDigest) + { + // Note that we can only get there if version == 3.0, which is the current_version. When we'll change the + // version, we'll have to deserialize/re-serialize the data to be in the proper version. + assert version == MessagingService.VERSION_30; + ByteBuffer data = ((DataResponse)response).data; + ByteBufferUtil.writeWithLength(data, out); + } + } + + public ReadResponse deserialize(DataInput in, int version) throws IOException + { + if (version < MessagingService.VERSION_30) + { + // TODO + throw new UnsupportedOperationException(); + } + + ByteBuffer digest = ByteBufferUtil.readWithShortLength(in); + if (digest.hasRemaining()) + return new DigestResponse(digest); + + assert version == MessagingService.VERSION_30; + ByteBuffer data = ByteBufferUtil.readWithLength(in); + return new DataResponse(data); + } + + public long serializedSize(ReadResponse response, int version) + { + if (version < MessagingService.VERSION_30) + { + // TODO + throw new UnsupportedOperationException(); + } + + TypeSizes sizes = TypeSizes.NATIVE; + boolean isDigest = response.isDigestQuery(); + long size = ByteBufferUtil.serializedSizeWithShortLength(isDigest ? response.digest() : ByteBufferUtil.EMPTY_BYTE_BUFFER, sizes); + + if (!isDigest) + { + // Note that we can only get there if version == 3.0, which is the current_version. When we'll change the + // version, we'll have to deserialize/re-serialize the data to be in the proper version. + assert version == MessagingService.VERSION_30; + ByteBuffer data = ((DataResponse)response).data; + size += ByteBufferUtil.serializedSizeWithLength(data, sizes); + } + return size; + } + } + + private static class LegacyRangeSliceReplySerializer implements IVersionedSerializer + { + public void serialize(ReadResponse response, DataOutputPlus out, int version) throws IOException + { + // TODO + throw new UnsupportedOperationException(); + // out.writeInt(rsr.rows.size()); + // for (Row row : rsr.rows) + // Row.serializer.serialize(row, out, version); + } + + public ReadResponse deserialize(DataInput in, int version) throws IOException + { + // TODO + throw new UnsupportedOperationException(); + // int rowCount = in.readInt(); + // List rows = new ArrayList(rowCount); + // for (int i = 0; i < rowCount; i++) + // rows.add(Row.serializer.deserialize(in, version)); + // return new RangeSliceReply(rows); + } + + public long serializedSize(ReadResponse response, int version) + { + // TODO + throw new UnsupportedOperationException(); + // int size = TypeSizes.NATIVE.sizeof(rsr.rows.size()); + // for (Row row : rsr.rows) + // size += Row.serializer.serializedSize(row, version); + // return size; + } } } diff --git a/src/java/org/apache/cassandra/db/RetriedSliceFromReadCommand.java b/src/java/org/apache/cassandra/db/RetriedSliceFromReadCommand.java deleted file mode 100644 index 41f5a50c0c..0000000000 --- a/src/java/org/apache/cassandra/db/RetriedSliceFromReadCommand.java +++ /dev/null @@ -1,56 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.apache.cassandra.db; - -import java.nio.ByteBuffer; - -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; - -import org.apache.cassandra.db.filter.SliceQueryFilter; - -public class RetriedSliceFromReadCommand extends SliceFromReadCommand -{ - static final Logger logger = LoggerFactory.getLogger(RetriedSliceFromReadCommand.class); - public final int originalCount; - - public RetriedSliceFromReadCommand(String keyspaceName, ByteBuffer key, String cfName, long timestamp, SliceQueryFilter filter, int originalCount) - { - super(keyspaceName, key, cfName, timestamp, filter); - this.originalCount = originalCount; - } - - @Override - public ReadCommand copy() - { - return new RetriedSliceFromReadCommand(ksName, key, cfName, timestamp, filter, originalCount).setIsDigestQuery(isDigestQuery()); - } - - @Override - public int getOriginalRequestedCount() - { - return originalCount; - } - - @Override - public String toString() - { - return "RetriedSliceFromReadCommand(" + "cmd=" + super.toString() + ", originalCount=" + originalCount + ")"; - } - -} diff --git a/src/java/org/apache/cassandra/db/ReusableClustering.java b/src/java/org/apache/cassandra/db/ReusableClustering.java new file mode 100644 index 0000000000..e2760aa408 --- /dev/null +++ b/src/java/org/apache/cassandra/db/ReusableClustering.java @@ -0,0 +1,82 @@ + +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.cassandra.db; + +import java.nio.ByteBuffer; +import java.util.Arrays; + +import org.apache.cassandra.utils.ObjectSizes; + +public class ReusableClustering extends Clustering +{ + private static final long EMPTY_SIZE = ObjectSizes.measure(new ReusableClustering(0)); + + protected final ByteBuffer[] values; + + protected ReusableWriter writer; + + public ReusableClustering(int size) + { + this.values = new ByteBuffer[size]; + } + + public int size() + { + return values.length; + } + + public ByteBuffer get(int i) + { + return values[i]; + } + + public ByteBuffer[] getRawValues() + { + return values; + } + + public Writer writer() + { + if (writer == null) + writer = new ReusableWriter(); + return writer; + } + + public void reset() + { + Arrays.fill(values, null); + if (writer != null) + writer.reset(); + } + + protected class ReusableWriter implements Writer + { + int idx; + + public void writeClusteringValue(ByteBuffer value) + { + values[idx++] = value; + } + + private void reset() + { + idx = 0; + } + } +} diff --git a/src/java/org/apache/cassandra/db/ReusableClusteringPrefix.java b/src/java/org/apache/cassandra/db/ReusableClusteringPrefix.java new file mode 100644 index 0000000000..d2f19f7112 --- /dev/null +++ b/src/java/org/apache/cassandra/db/ReusableClusteringPrefix.java @@ -0,0 +1,57 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.cassandra.db; + +import java.nio.ByteBuffer; +import java.util.Arrays; + +import org.apache.cassandra.utils.ObjectSizes; + +// Note that we abuse a bit ReusableClustering to store Slice.Bound infos, but it's convenient so ... +public class ReusableClusteringPrefix extends ReusableClustering +{ + private Kind kind; + private int size; + + public ReusableClusteringPrefix(int size) + { + super(size); + } + + public ClusteringPrefix get() + { + // We use ReusableClusteringPrefix when writing sstables (in ColumnIndex) and we + // don't write static clustering there. + assert kind != Kind.STATIC_CLUSTERING; + if (kind == Kind.CLUSTERING) + { + assert values.length == size; + return this; + } + + return Slice.Bound.create(kind, Arrays.copyOfRange(values, 0, size)); + } + + public void copy(ClusteringPrefix clustering) + { + kind = clustering.kind(); + for (int i = 0; i < clustering.size(); i++) + values[i] = clustering.get(i); + size = clustering.size(); + } +} diff --git a/src/java/org/apache/cassandra/db/ReusableLivenessInfo.java b/src/java/org/apache/cassandra/db/ReusableLivenessInfo.java new file mode 100644 index 0000000000..43530b06f5 --- /dev/null +++ b/src/java/org/apache/cassandra/db/ReusableLivenessInfo.java @@ -0,0 +1,65 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.cassandra.db; + +public class ReusableLivenessInfo extends AbstractLivenessInfo +{ + private long timestamp; + private int ttl; + private int localDeletionTime; + + public ReusableLivenessInfo() + { + reset(); + } + + public LivenessInfo setTo(LivenessInfo info) + { + return setTo(info.timestamp(), info.ttl(), info.localDeletionTime()); + } + + public LivenessInfo setTo(long timestamp, int ttl, int localDeletionTime) + { + this.timestamp = timestamp; + this.ttl = ttl; + this.localDeletionTime = localDeletionTime; + return this; + } + + public long timestamp() + { + return timestamp; + } + + public int ttl() + { + return ttl; + } + + public int localDeletionTime() + { + return localDeletionTime; + } + + public void reset() + { + this.timestamp = LivenessInfo.NO_TIMESTAMP; + this.ttl = LivenessInfo.NO_TTL; + this.localDeletionTime = LivenessInfo.NO_DELETION_TIME; + } +} diff --git a/src/java/org/apache/cassandra/db/Row.java b/src/java/org/apache/cassandra/db/Row.java deleted file mode 100644 index a8268946c2..0000000000 --- a/src/java/org/apache/cassandra/db/Row.java +++ /dev/null @@ -1,88 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.apache.cassandra.db; - -import java.io.*; -import java.nio.ByteBuffer; - -import org.apache.cassandra.db.filter.IDiskAtomFilter; -import org.apache.cassandra.io.IVersionedSerializer; -import org.apache.cassandra.io.util.DataOutputPlus; -import org.apache.cassandra.service.StorageService; -import org.apache.cassandra.utils.ByteBufferUtil; - -public class Row -{ - public static final RowSerializer serializer = new RowSerializer(); - - public final DecoratedKey key; - public final ColumnFamily cf; - - public Row(DecoratedKey key, ColumnFamily cf) - { - assert key != null; - // cf may be null, indicating no data - this.key = key; - this.cf = cf; - } - - public Row(ByteBuffer key, ColumnFamily updates) - { - this(StorageService.getPartitioner().decorateKey(key), updates); - } - - @Override - public String toString() - { - return "Row(" + - "key=" + key + - ", cf=" + cf + - ')'; - } - - public int getLiveCount(IDiskAtomFilter filter, long now) - { - return cf == null ? 0 : filter.getLiveCount(cf, now); - } - - public static class RowSerializer implements IVersionedSerializer - { - public void serialize(Row row, DataOutputPlus out, int version) throws IOException - { - ByteBufferUtil.writeWithShortLength(row.key.getKey(), out); - ColumnFamily.serializer.serialize(row.cf, out, version); - } - - public Row deserialize(DataInput in, int version, ColumnSerializer.Flag flag) throws IOException - { - return new Row(StorageService.getPartitioner().decorateKey(ByteBufferUtil.readWithShortLength(in)), - ColumnFamily.serializer.deserialize(in, flag, version)); - } - - public Row deserialize(DataInput in, int version) throws IOException - { - return deserialize(in, version, ColumnSerializer.Flag.LOCAL); - } - - public long serializedSize(Row row, int version) - { - int keySize = row.key.getKey().remaining(); - return TypeSizes.NATIVE.sizeof((short) keySize) + keySize + ColumnFamily.serializer.serializedSize(row.cf, TypeSizes.NATIVE, version); - } - } -} diff --git a/src/java/org/apache/cassandra/db/RowIndexEntry.java b/src/java/org/apache/cassandra/db/RowIndexEntry.java index 4ff61cee7e..016e26e5c6 100644 --- a/src/java/org/apache/cassandra/db/RowIndexEntry.java +++ b/src/java/org/apache/cassandra/db/RowIndexEntry.java @@ -26,6 +26,7 @@ import java.util.List; import com.google.common.primitives.Ints; +import org.apache.cassandra.config.CFMetaData; import org.apache.cassandra.cache.IMeasurableMemory; import org.apache.cassandra.io.ISerializer; import org.apache.cassandra.io.sstable.IndexHelper; @@ -45,7 +46,7 @@ public class RowIndexEntry implements IMeasurableMemory this.position = position; } - public int promotedSize(ISerializer idxSerializer) + public int promotedSize(CFMetaData metadata, Version version, SerializationHeader header) { return 0; } @@ -100,34 +101,39 @@ public class RowIndexEntry implements IMeasurableMemory public static interface IndexSerializer { void serialize(RowIndexEntry rie, DataOutputPlus out) throws IOException; - RowIndexEntry deserialize(DataInput in, Version version) throws IOException; + RowIndexEntry deserialize(DataInput in) throws IOException; public int serializedSize(RowIndexEntry rie); } public static class Serializer implements IndexSerializer { - private final ISerializer idxSerializer; + private final CFMetaData metadata; + private final Version version; + private final SerializationHeader header; - public Serializer(ISerializer idxSerializer) + public Serializer(CFMetaData metadata, Version version, SerializationHeader header) { - this.idxSerializer = idxSerializer; + this.metadata = metadata; + this.version = version; + this.header = header; } public void serialize(RowIndexEntry rie, DataOutputPlus out) throws IOException { out.writeLong(rie.position); - out.writeInt(rie.promotedSize(idxSerializer)); + out.writeInt(rie.promotedSize(metadata, version, header)); if (rie.isIndexed()) { DeletionTime.serializer.serialize(rie.deletionTime(), out); out.writeInt(rie.columnsIndex().size()); + IndexHelper.IndexInfo.Serializer idxSerializer = metadata.serializers().indexSerializer(version); for (IndexHelper.IndexInfo info : rie.columnsIndex()) - idxSerializer.serialize(info, out); + idxSerializer.serialize(info, out, header); } } - public RowIndexEntry deserialize(DataInput in, Version version) throws IOException + public RowIndexEntry deserialize(DataInput in) throws IOException { long position = in.readLong(); @@ -137,9 +143,10 @@ public class RowIndexEntry implements IMeasurableMemory DeletionTime deletionTime = DeletionTime.serializer.deserialize(in); int entries = in.readInt(); + IndexHelper.IndexInfo.Serializer idxSerializer = metadata.serializers().indexSerializer(version); List columnsIndex = new ArrayList<>(entries); for (int i = 0; i < entries; i++) - columnsIndex.add(idxSerializer.deserialize(in)); + columnsIndex.add(idxSerializer.deserialize(in, header)); return new IndexedEntry(position, deletionTime, columnsIndex); } @@ -166,7 +173,7 @@ public class RowIndexEntry implements IMeasurableMemory public int serializedSize(RowIndexEntry rie) { - int size = TypeSizes.NATIVE.sizeof(rie.position) + TypeSizes.NATIVE.sizeof(rie.promotedSize(idxSerializer)); + int size = TypeSizes.NATIVE.sizeof(rie.position) + TypeSizes.NATIVE.sizeof(rie.promotedSize(metadata, version, header)); if (rie.isIndexed()) { @@ -175,8 +182,9 @@ public class RowIndexEntry implements IMeasurableMemory size += DeletionTime.serializer.serializedSize(rie.deletionTime(), TypeSizes.NATIVE); size += TypeSizes.NATIVE.sizeof(index.size()); + IndexHelper.IndexInfo.Serializer idxSerializer = metadata.serializers().indexSerializer(version); for (IndexHelper.IndexInfo info : index) - size += idxSerializer.serializedSize(info, TypeSizes.NATIVE); + size += idxSerializer.serializedSize(info, header, TypeSizes.NATIVE); } @@ -217,13 +225,14 @@ public class RowIndexEntry implements IMeasurableMemory } @Override - public int promotedSize(ISerializer idxSerializer) + public int promotedSize(CFMetaData metadata, Version version, SerializationHeader header) { TypeSizes typeSizes = TypeSizes.NATIVE; long size = DeletionTime.serializer.serializedSize(deletionTime, typeSizes); size += typeSizes.sizeof(columnsIndex.size()); // number of entries + IndexHelper.IndexInfo.Serializer idxSerializer = metadata.serializers().indexSerializer(version); for (IndexHelper.IndexInfo info : columnsIndex) - size += idxSerializer.serializedSize(info, typeSizes); + size += idxSerializer.serializedSize(info, header, typeSizes); return Ints.checkedCast(size); } diff --git a/src/java/org/apache/cassandra/db/RowIteratorFactory.java b/src/java/org/apache/cassandra/db/RowIteratorFactory.java deleted file mode 100644 index 3473e96faa..0000000000 --- a/src/java/org/apache/cassandra/db/RowIteratorFactory.java +++ /dev/null @@ -1,172 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.apache.cassandra.db; - -import java.util.*; - -import com.google.common.collect.Iterables; - -import org.apache.cassandra.db.columniterator.IColumnIteratorFactory; -import org.apache.cassandra.db.columniterator.LazyColumnIterator; -import org.apache.cassandra.db.columniterator.OnDiskAtomIterator; -import org.apache.cassandra.db.filter.QueryFilter; -import org.apache.cassandra.db.filter.IDiskAtomFilter; -import org.apache.cassandra.db.filter.TombstoneOverwhelmingException; -import org.apache.cassandra.io.sstable.format.SSTableReader; -import org.apache.cassandra.utils.CloseableIterator; -import org.apache.cassandra.utils.MergeIterator; - -public class RowIteratorFactory -{ - - private static final Comparator COMPARE_BY_KEY = new Comparator() - { - public int compare(OnDiskAtomIterator o1, OnDiskAtomIterator o2) - { - return DecoratedKey.comparator.compare(o1.getKey(), o2.getKey()); - } - }; - - - /** - * Get a row iterator over the provided memtables and sstables, between the provided keys - * and filtered by the queryfilter. - * @param memtables Memtables pending flush. - * @param sstables SStables to scan through. - * @param range The data range to fetch - * @param cfs - * @return A row iterator following all the given restrictions - */ - public static CloseableIterator getIterator(final Iterable memtables, - final Collection sstables, - final DataRange range, - final ColumnFamilyStore cfs, - final long now) - { - // fetch data from current memtable, historical memtables, and SSTables in the correct order. - final List> iterators = new ArrayList<>(Iterables.size(memtables) + sstables.size()); - - for (Memtable memtable : memtables) - iterators.add(new ConvertToColumnIterator(range, memtable.getEntryIterator(range.startKey(), range.stopKey()))); - - for (SSTableReader sstable : sstables) - iterators.add(sstable.getScanner(range)); - - // reduce rows from all sources into a single row - return MergeIterator.get(iterators, COMPARE_BY_KEY, new MergeIterator.Reducer() - { - private final int gcBefore = cfs.gcBefore(now); - private final List colIters = new ArrayList<>(); - private DecoratedKey key; - private ColumnFamily returnCF; - - @Override - protected void onKeyChange() - { - this.returnCF = ArrayBackedSortedColumns.factory.create(cfs.metadata, range.columnFilter.isReversed()); - } - - public void reduce(OnDiskAtomIterator current) - { - this.colIters.add(current); - this.key = current.getKey(); - this.returnCF.delete(current.getColumnFamily()); - } - - protected Row getReduced() - { - // First check if this row is in the rowCache. If it is and it covers our filter, we can skip the rest - ColumnFamily cached = cfs.getRawCachedRow(key); - IDiskAtomFilter filter = range.columnFilter(key.getKey()); - - try - { - if (cached == null || !cfs.isFilterFullyCoveredBy(filter, cached, now)) - { - // not cached: collate - QueryFilter.collateOnDiskAtom(returnCF, colIters, filter, key, gcBefore, now); - } - else - { - QueryFilter keyFilter = new QueryFilter(key, cfs.name, filter, now); - returnCF = cfs.filterColumnFamily(cached, keyFilter); - } - } - catch(TombstoneOverwhelmingException e) - { - e.setKey(key); - throw e; - } - - Row rv = new Row(key, returnCF); - colIters.clear(); - key = null; - return rv; - } - }); - } - - /** - * Get a ColumnIterator for a specific key in the memtable. - */ - private static class ConvertToColumnIterator implements CloseableIterator - { - private final DataRange range; - private final Iterator> iter; - - public ConvertToColumnIterator(DataRange range, Iterator> iter) - { - this.range = range; - this.iter = iter; - } - - public boolean hasNext() - { - return iter.hasNext(); - } - - /* - * Note that when doing get_paged_slice, we reset the start of the queryFilter after we've fetched the - * first row. This means that this iterator should not use in any way the filter to fetch a row before - * we call next(). Which prevents us for using guava AbstractIterator. - * This is obviously rather fragile and we should consider refactoring that code, but such refactor will go - * deep into the storage engine code so this will have to do until then. - */ - public OnDiskAtomIterator next() - { - final Map.Entry entry = iter.next(); - return new LazyColumnIterator(entry.getKey(), new IColumnIteratorFactory() - { - public OnDiskAtomIterator create() - { - return range.columnFilter(entry.getKey().getKey()).getColumnIterator(entry.getKey(), entry.getValue()); - } - }); - } - - public void remove() - { - throw new UnsupportedOperationException(); - } - - public void close() - { - // pass - } - } -} diff --git a/src/java/org/apache/cassandra/db/RowUpdateBuilder.java b/src/java/org/apache/cassandra/db/RowUpdateBuilder.java new file mode 100644 index 0000000000..c3f3d292b4 --- /dev/null +++ b/src/java/org/apache/cassandra/db/RowUpdateBuilder.java @@ -0,0 +1,316 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.cassandra.db; + +import java.nio.ByteBuffer; + +import org.apache.cassandra.cql3.ColumnIdentifier; +import org.apache.cassandra.config.CFMetaData; +import org.apache.cassandra.config.ColumnDefinition; +import org.apache.cassandra.db.rows.*; +import org.apache.cassandra.db.context.CounterContext; +import org.apache.cassandra.db.partitions.*; +import org.apache.cassandra.db.marshal.AbstractType; +import org.apache.cassandra.db.marshal.ListType; +import org.apache.cassandra.db.marshal.MapType; +import org.apache.cassandra.service.StorageService; +import org.apache.cassandra.utils.*; + +/** + * Convenience object to create single row updates. + * + * This is meant for system table update, when performance is not of the utmost importance. + */ +public class RowUpdateBuilder +{ + private final PartitionUpdate update; + + private final LivenessInfo defaultLiveness; + private final LivenessInfo deletionLiveness; + private final DeletionTime deletionTime; + + private final Mutation mutation; + + private Row.Writer regularWriter; + private Row.Writer staticWriter; + + private boolean hasSetClustering; + private boolean useRowMarker = true; + + private RowUpdateBuilder(PartitionUpdate update, long timestamp, int ttl, int localDeletionTime, Mutation mutation) + { + this.update = update; + + this.defaultLiveness = SimpleLivenessInfo.forUpdate(timestamp, ttl, localDeletionTime, update.metadata()); + this.deletionLiveness = SimpleLivenessInfo.forDeletion(timestamp, localDeletionTime); + this.deletionTime = new SimpleDeletionTime(timestamp, localDeletionTime); + + // note that the created mutation may get further update later on, so we don't use the ctor that create a singletonMap + // underneath (this class if for convenience, not performance) + this.mutation = mutation == null ? new Mutation(update.metadata().ksName, update.partitionKey()).add(update) : mutation; + } + + private RowUpdateBuilder(PartitionUpdate update, long timestamp, int ttl, Mutation mutation) + { + this(update, timestamp, ttl, FBUtilities.nowInSeconds(), mutation); + } + + private Row.Writer writer() + { + assert staticWriter == null : "Cannot update both static and non-static columns with the same RowUpdateBuilder object"; + if (regularWriter == null) + { + regularWriter = update.writer(); + + // If a CQL table, add the "row marker" + if (update.metadata().isCQLTable() && useRowMarker) + regularWriter.writePartitionKeyLivenessInfo(defaultLiveness); + } + return regularWriter; + } + + private Row.Writer staticWriter() + { + assert regularWriter == null : "Cannot update both static and non-static columns with the same RowUpdateBuilder object"; + if (staticWriter == null) + staticWriter = update.staticWriter(); + return staticWriter; + } + + private Row.Writer writer(ColumnDefinition c) + { + return c.isStatic() ? staticWriter() : writer(); + } + + public RowUpdateBuilder(CFMetaData metadata, long timestamp, Object partitionKey) + { + this(metadata, FBUtilities.nowInSeconds(), timestamp, partitionKey); + } + + public RowUpdateBuilder(CFMetaData metadata, int localDeletionTime, long timestamp, Object partitionKey) + { + this(metadata, localDeletionTime, timestamp, metadata.getDefaultTimeToLive(), partitionKey); + } + + public RowUpdateBuilder(CFMetaData metadata, long timestamp, int ttl, Object partitionKey) + { + this(metadata, FBUtilities.nowInSeconds(), timestamp, ttl, partitionKey); + } + + public RowUpdateBuilder(CFMetaData metadata, int localDeletionTime, long timestamp, int ttl, Object partitionKey) + { + this(new PartitionUpdate(metadata, makeKey(metadata, partitionKey), metadata.partitionColumns(), 1), timestamp, ttl, localDeletionTime, null); + } + + public RowUpdateBuilder(CFMetaData metadata, long timestamp, Mutation mutation) + { + this(metadata, timestamp, LivenessInfo.NO_TTL, mutation); + } + + public RowUpdateBuilder(CFMetaData metadata, long timestamp, int ttl, Mutation mutation) + { + this(getOrAdd(metadata, mutation), timestamp, ttl, mutation); + } + + public RowUpdateBuilder(PartitionUpdate update, long timestamp, int ttl) + { + this(update, timestamp, ttl, null); + } + + // This must be called before any addition or deletion if used. + public RowUpdateBuilder noRowMarker() + { + this.useRowMarker = false; + return this; + } + + public RowUpdateBuilder clustering(Object... clusteringValues) + { + assert clusteringValues.length == update.metadata().comparator.size() + : "Invalid clustering values length. Expected: " + update.metadata().comparator.size() + " got: " + clusteringValues.length; + hasSetClustering = true; + if (clusteringValues.length > 0) + Rows.writeClustering(update.metadata().comparator.make(clusteringValues), writer()); + return this; + } + + public Mutation build() + { + Row.Writer writer = regularWriter == null ? staticWriter : regularWriter; + if (writer != null) + writer.endOfRow(); + return mutation; + } + + public PartitionUpdate buildUpdate() + { + build(); + return update; + } + + private static void deleteRow(PartitionUpdate update, long timestamp, Object...clusteringValues) + { + assert clusteringValues.length == update.metadata().comparator.size() || (clusteringValues.length == 0 && !update.columns().statics.isEmpty()); + + Row.Writer writer = clusteringValues.length == update.metadata().comparator.size() + ? update.writer() + : update.staticWriter(); + + if (clusteringValues.length > 0) + Rows.writeClustering(update.metadata().comparator.make(clusteringValues), writer); + writer.writeRowDeletion(new SimpleDeletionTime(timestamp, FBUtilities.nowInSeconds())); + writer.endOfRow(); + } + + public static Mutation deleteRow(CFMetaData metadata, long timestamp, Mutation mutation, Object... clusteringValues) + { + deleteRow(getOrAdd(metadata, mutation), timestamp, clusteringValues); + return mutation; + } + + public static Mutation deleteRow(CFMetaData metadata, long timestamp, Object key, Object... clusteringValues) + { + PartitionUpdate update = new PartitionUpdate(metadata, makeKey(metadata, key), metadata.partitionColumns(), 0); + deleteRow(update, timestamp, clusteringValues); + // note that the created mutation may get further update later on, so we don't use the ctor that create a singletonMap + // underneath (this class if for convenience, not performance) + return new Mutation(update.metadata().ksName, update.partitionKey()).add(update); + } + + private static DecoratedKey makeKey(CFMetaData metadata, Object... partitionKey) + { + if (partitionKey.length == 1 && partitionKey[0] instanceof DecoratedKey) + return (DecoratedKey)partitionKey[0]; + + ByteBuffer key = CFMetaData.serializePartitionKey(metadata.getKeyValidatorAsClusteringComparator().make(partitionKey)); + return StorageService.getPartitioner().decorateKey(key); + } + + private static PartitionUpdate getOrAdd(CFMetaData metadata, Mutation mutation) + { + PartitionUpdate upd = mutation.get(metadata); + if (upd == null) + { + upd = new PartitionUpdate(metadata, mutation.key(), metadata.partitionColumns(), 1); + mutation.add(upd); + } + return upd; + } + + public RowUpdateBuilder resetCollection(String columnName) + { + ColumnDefinition c = getDefinition(columnName); + assert c != null : "Cannot find column " + columnName; + assert c.isStatic() || update.metadata().comparator.size() == 0 || hasSetClustering : "Cannot set non static column " + c + " since no clustering hasn't been provided"; + assert c.type.isCollection() && c.type.isMultiCell(); + writer(c).writeComplexDeletion(c, new SimpleDeletionTime(defaultLiveness.timestamp() - 1, deletionTime.localDeletionTime())); + return this; + } + + public RowUpdateBuilder addRangeTombstone(RangeTombstone rt) + { + update.addRangeTombstone(rt); + return this; + } + + public RowUpdateBuilder addRangeTombstone(Slice slice) + { + update.addRangeTombstone(slice, deletionTime); + return this; + } + + public RowUpdateBuilder addRangeTombstone(Object start, Object end) + { + ClusteringComparator cmp = update.metadata().comparator; + Slice slice = Slice.make(cmp.make(start), cmp.make(end)); + return addRangeTombstone(slice); + } + + public RowUpdateBuilder add(String columnName, Object value) + { + ColumnDefinition c = getDefinition(columnName); + assert c != null : "Cannot find column " + columnName; + return add(c, value); + } + + public RowUpdateBuilder add(ColumnDefinition columnDefinition, Object value) + { + assert columnDefinition.isStatic() || update.metadata().comparator.size() == 0 || hasSetClustering : "Cannot set non static column " + columnDefinition + " since no clustering hasn't been provided"; + if (value == null) + writer(columnDefinition).writeCell(columnDefinition, false, ByteBufferUtil.EMPTY_BYTE_BUFFER, deletionLiveness, null); + else + writer(columnDefinition).writeCell(columnDefinition, false, bb(value, columnDefinition.type), defaultLiveness, null); + return this; + } + + public RowUpdateBuilder delete(String columnName) + { + ColumnDefinition c = getDefinition(columnName); + assert c != null : "Cannot find column " + columnName; + return delete(c); + } + + public RowUpdateBuilder delete(ColumnDefinition columnDefinition) + { + return add(columnDefinition, null); + } + + private ByteBuffer bb(Object value, AbstractType type) + { + if (value instanceof ByteBuffer) + return (ByteBuffer)value; + + if (type.isCounter()) + { + // See UpdateParameters.addCounter() + assert value instanceof Long : "Attempted to adjust Counter cell with non-long value."; + return CounterContext.instance().createGlobal(CounterId.getLocalId(), 1, (Long)value); + } + return ((AbstractType)type).decompose(value); + } + + public RowUpdateBuilder addMapEntry(String columnName, Object key, Object value) + { + ColumnDefinition c = getDefinition(columnName); + assert c.isStatic() || update.metadata().comparator.size() == 0 || hasSetClustering : "Cannot set non static column " + c + " since no clustering hasn't been provided"; + assert c.type instanceof MapType; + MapType mt = (MapType)c.type; + writer(c).writeCell(c, false, bb(value, mt.getValuesType()), defaultLiveness, CellPath.create(bb(key, mt.getKeysType()))); + return this; + } + + public RowUpdateBuilder addListEntry(String columnName, Object value) + { + ColumnDefinition c = getDefinition(columnName); + assert c.isStatic() || hasSetClustering : "Cannot set non static column " + c + " since no clustering hasn't been provided"; + assert c.type instanceof ListType; + ListType lt = (ListType)c.type; + writer(c).writeCell(c, false, bb(value, lt.getElementsType()), defaultLiveness, CellPath.create(ByteBuffer.wrap(UUIDGen.getTimeUUIDBytes()))); + return this; + } + + private ColumnDefinition getDefinition(String name) + { + return update.metadata().getColumnDefinition(new ColumnIdentifier(name, true)); + } + + public UnfilteredRowIterator unfilteredIterator() + { + return update.unfilteredIterator(); + } +} diff --git a/src/java/org/apache/cassandra/db/SerializationHeader.java b/src/java/org/apache/cassandra/db/SerializationHeader.java new file mode 100644 index 0000000000..304332ebba --- /dev/null +++ b/src/java/org/apache/cassandra/db/SerializationHeader.java @@ -0,0 +1,488 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.cassandra.db; + +import java.io.DataInput; +import java.io.IOException; +import java.nio.ByteBuffer; +import java.util.*; + +import com.google.common.collect.ImmutableList; +import com.google.common.collect.Lists; +import com.google.common.base.Function; + +import org.apache.cassandra.config.CFMetaData; +import org.apache.cassandra.config.ColumnDefinition; +import org.apache.cassandra.db.rows.*; +import org.apache.cassandra.db.marshal.AbstractType; +import org.apache.cassandra.db.marshal.BytesType; +import org.apache.cassandra.db.marshal.UTF8Type; +import org.apache.cassandra.db.marshal.TypeParser; +import org.apache.cassandra.io.sstable.format.SSTableReader; +import org.apache.cassandra.io.sstable.format.Version; +import org.apache.cassandra.io.sstable.metadata.MetadataType; +import org.apache.cassandra.io.sstable.metadata.MetadataComponent; +import org.apache.cassandra.io.sstable.metadata.IMetadataComponentSerializer; +import org.apache.cassandra.io.util.DataOutputPlus; +import org.apache.cassandra.utils.ByteBufferUtil; + +public class SerializationHeader +{ + private static final int DEFAULT_BASE_DELETION = computeDefaultBaseDeletion(); + + public static final Serializer serializer = new Serializer(); + + private final AbstractType keyType; + private final List> clusteringTypes; + + private final PartitionColumns columns; + private final RowStats stats; + + private final Map> typeMap; + + private final long baseTimestamp; + public final int baseDeletionTime; + private final int baseTTL; + + // Whether or not to store cell in a sparse or dense way. See UnfilteredSerializer for details. + private final boolean useSparseColumnLayout; + + private SerializationHeader(AbstractType keyType, + List> clusteringTypes, + PartitionColumns columns, + RowStats stats, + Map> typeMap) + { + this.keyType = keyType; + this.clusteringTypes = clusteringTypes; + this.columns = columns; + this.stats = stats; + this.typeMap = typeMap; + + // Not that if a given stats is unset, it means that either it's unused (there is + // no tombstone whatsoever for instance) or that we have no information on it. In + // that former case, it doesn't matter which base we use but in the former, we use + // bases that are more likely to provide small encoded values than the default + // "unset" value. + this.baseTimestamp = stats.hasMinTimestamp() ? stats.minTimestamp : 0; + this.baseDeletionTime = stats.hasMinLocalDeletionTime() ? stats.minLocalDeletionTime : DEFAULT_BASE_DELETION; + this.baseTTL = stats.minTTL; + + // For the dense layout, we have a 1 byte overhead for absent columns. For the sparse layout, it's a 1 + // overhead for present columns (in fact we use a 2 byte id, but assuming vint encoding, we'll pay 2 bytes + // only for the columns after the 128th one and for simplicity we assume that once you have that many column, + // you'll tend towards a clearly dense or clearly sparse case so that the heurstic above shouldn't still be + // too inapropriate). So if on average more than half of our columns are set per row, we better go for sparse. + this.useSparseColumnLayout = stats.avgColumnSetPerRow <= (columns.regulars.columnCount()/ 2); + } + + public boolean useSparseColumnLayout(boolean isStatic) + { + // We always use a dense layout for the static row. Having very many static columns with only a few set at + // any given time doesn't feel very common at all (and we already optimize the case where no static at all + // are provided). + return isStatic ? false : useSparseColumnLayout; + } + + public static SerializationHeader forKeyCache(CFMetaData metadata) + { + // We don't save type information in the key cache (we could change + // that but it's easier right now), so instead we simply use BytesType + // for both serialization and deserialization. Note that we also only + // serializer clustering prefixes in the key cache, so only the clusteringTypes + // really matter. + int size = metadata.clusteringColumns().size(); + List> clusteringTypes = new ArrayList<>(size); + for (int i = 0; i < size; i++) + clusteringTypes.add(BytesType.instance); + return new SerializationHeader(BytesType.instance, + clusteringTypes, + PartitionColumns.NONE, + RowStats.NO_STATS, + Collections.>emptyMap()); + } + + public static SerializationHeader make(CFMetaData metadata, Collection sstables) + { + // The serialization header has to be computed before the start of compaction (since it's used to write) + // the result. This means that when compacting multiple sources, we won't have perfectly accurate stats + // (for RowStats) since compaction may delete, purge and generally merge rows in unknown ways. This is + // kind of ok because those stats are only used for optimizing the underlying storage format and so we + // just have to strive for as good as possible. Currently, we stick to a relatively naive merge of existing + // global stats because it's simple and probably good enough in most situation but we could probably + // improve our marging of inaccuracy through the use of more fine-grained stats in the future. + // Note however that to avoid seeing our accuracy degrade through successive compactions, we don't base + // our stats merging on the compacted files headers, which as we just said can be somewhat inaccurate, + // but rather on their stats stored in StatsMetadata that are fully accurate. + RowStats.Collector stats = new RowStats.Collector(); + PartitionColumns.Builder columns = PartitionColumns.builder(); + for (SSTableReader sstable : sstables) + { + stats.updateTimestamp(sstable.getMinTimestamp()); + stats.updateLocalDeletionTime(sstable.getMinLocalDeletionTime()); + stats.updateTTL(sstable.getMinTTL()); + stats.updateColumnSetPerRow(sstable.getTotalColumnsSet(), sstable.getTotalRows()); + if (sstable.header == null) + columns.addAll(metadata.partitionColumns()); + else + columns.addAll(sstable.header.columns()); + } + return new SerializationHeader(metadata, columns.build(), stats.get()); + } + + public SerializationHeader(CFMetaData metadata, + PartitionColumns columns, + RowStats stats) + { + this(metadata.getKeyValidator(), + typesOf(metadata.clusteringColumns()), + columns, + stats, + null); + } + + private static List> typesOf(List columns) + { + return ImmutableList.copyOf(Lists.transform(columns, new Function>() + { + public AbstractType apply(ColumnDefinition column) + { + return column.type; + } + })); + } + + public PartitionColumns columns() + { + return columns; + } + + public boolean hasStatic() + { + return !columns.statics.isEmpty(); + } + + private static int computeDefaultBaseDeletion() + { + // We need a fixed default, but one that is likely to provide small values (close to 0) when + // substracted to deletion times. Since deletion times are 'the current time in seconds', we + // use as base Jan 1, 2015 (in seconds). + Calendar c = Calendar.getInstance(TimeZone.getTimeZone("GMT-0"), Locale.US); + c.set(Calendar.YEAR, 2015); + c.set(Calendar.MONTH, Calendar.JANUARY); + c.set(Calendar.DAY_OF_MONTH, 1); + c.set(Calendar.HOUR_OF_DAY, 0); + c.set(Calendar.MINUTE, 0); + c.set(Calendar.SECOND, 0); + c.set(Calendar.MILLISECOND, 0); + return (int)(c.getTimeInMillis() / 1000); + } + + public RowStats stats() + { + return stats; + } + + public AbstractType keyType() + { + return keyType; + } + + public List> clusteringTypes() + { + return clusteringTypes; + } + + public Columns columns(boolean isStatic) + { + return isStatic ? columns.statics : columns.regulars; + } + + public AbstractType getType(ColumnDefinition column) + { + return typeMap == null ? column.type : typeMap.get(column.name.bytes); + } + + public long encodeTimestamp(long timestamp) + { + return timestamp - baseTimestamp; + } + + public long decodeTimestamp(long timestamp) + { + return baseTimestamp + timestamp; + } + + public int encodeDeletionTime(int deletionTime) + { + return deletionTime - baseDeletionTime; + } + + public int decodeDeletionTime(int deletionTime) + { + return baseDeletionTime + deletionTime; + } + + public int encodeTTL(int ttl) + { + return ttl - baseTTL; + } + + public int decodeTTL(int ttl) + { + return baseTTL + ttl; + } + + public Component toComponent() + { + Map> staticColumns = new LinkedHashMap<>(); + Map> regularColumns = new LinkedHashMap<>(); + for (ColumnDefinition column : columns.statics) + staticColumns.put(column.name.bytes, column.type); + for (ColumnDefinition column : columns.regulars) + regularColumns.put(column.name.bytes, column.type); + return new Component(keyType, clusteringTypes, staticColumns, regularColumns, stats); + } + + @Override + public String toString() + { + return String.format("SerializationHeader[key=%s, cks=%s, columns=%s, stats=%s, typeMap=%s, baseTs=%d, baseDt=%s, baseTTL=%s]", + keyType, clusteringTypes, columns, stats, typeMap, baseTimestamp, baseDeletionTime, baseTTL); + } + + /** + * We need the CFMetadata to properly deserialize a SerializationHeader but it's clunky to pass that to + * a SSTable component, so we use this temporary object to delay the actual need for the metadata. + */ + public static class Component extends MetadataComponent + { + private final AbstractType keyType; + private final List> clusteringTypes; + private final Map> staticColumns; + private final Map> regularColumns; + private final RowStats stats; + + private Component(AbstractType keyType, + List> clusteringTypes, + Map> staticColumns, + Map> regularColumns, + RowStats stats) + { + this.keyType = keyType; + this.clusteringTypes = clusteringTypes; + this.staticColumns = staticColumns; + this.regularColumns = regularColumns; + this.stats = stats; + } + + public MetadataType getType() + { + return MetadataType.HEADER; + } + + public SerializationHeader toHeader(CFMetaData metadata) + { + Map> typeMap = new HashMap<>(staticColumns.size() + regularColumns.size()); + typeMap.putAll(staticColumns); + typeMap.putAll(regularColumns); + + PartitionColumns.Builder builder = PartitionColumns.builder(); + for (ByteBuffer name : typeMap.keySet()) + { + ColumnDefinition column = metadata.getColumnDefinition(name); + if (column == null) + { + // TODO: this imply we don't read data for a column we don't yet know about, which imply this is theoretically + // racy with column addition. Currently, it is up to the user to not write data before the schema has propagated + // and this is far from being the only place that has such problem in practice. This doesn't mean we shouldn't + // improve this. + + // If we don't find the definition, it could be we have data for a dropped column, and we shouldn't + // fail deserialization because of that. So we grab a "fake" ColumnDefinition that ensure proper + // deserialization. The column will be ignore later on anyway. + column = metadata.getDroppedColumnDefinition(name); + if (column == null) + throw new RuntimeException("Unknown column " + UTF8Type.instance.getString(name) + " during deserialization"); + } + builder.add(column); + } + return new SerializationHeader(keyType, clusteringTypes, builder.build(), stats, typeMap); + } + + @Override + public boolean equals(Object o) + { + if(!(o instanceof Component)) + return false; + + Component that = (Component)o; + return Objects.equals(this.keyType, that.keyType) + && Objects.equals(this.clusteringTypes, that.clusteringTypes) + && Objects.equals(this.staticColumns, that.staticColumns) + && Objects.equals(this.regularColumns, that.regularColumns) + && Objects.equals(this.stats, that.stats); + } + + @Override + public int hashCode() + { + return Objects.hash(keyType, clusteringTypes, staticColumns, regularColumns, stats); + } + + @Override + public String toString() + { + return String.format("SerializationHeader.Component[key=%s, cks=%s, statics=%s, regulars=%s, stats=%s]", + keyType, clusteringTypes, staticColumns, regularColumns, stats); + } + } + + public static class Serializer implements IMetadataComponentSerializer + { + public void serializeForMessaging(SerializationHeader header, DataOutputPlus out, boolean hasStatic) throws IOException + { + RowStats.serializer.serialize(header.stats, out); + + if (hasStatic) + Columns.serializer.serialize(header.columns.statics, out); + Columns.serializer.serialize(header.columns.regulars, out); + } + + public SerializationHeader deserializeForMessaging(DataInput in, CFMetaData metadata, boolean hasStatic) throws IOException + { + RowStats stats = RowStats.serializer.deserialize(in); + + AbstractType keyType = metadata.getKeyValidator(); + List> clusteringTypes = typesOf(metadata.clusteringColumns()); + + Columns statics = hasStatic ? Columns.serializer.deserialize(in, metadata) : Columns.NONE; + Columns regulars = Columns.serializer.deserialize(in, metadata); + + return new SerializationHeader(keyType, clusteringTypes, new PartitionColumns(statics, regulars), stats, null); + } + + public long serializedSizeForMessaging(SerializationHeader header, TypeSizes sizes, boolean hasStatic) + { + long size = RowStats.serializer.serializedSize(header.stats, sizes); + + if (hasStatic) + size += Columns.serializer.serializedSize(header.columns.statics, sizes); + size += Columns.serializer.serializedSize(header.columns.regulars, sizes); + return size; + } + + // For SSTables + public void serialize(Component header, DataOutputPlus out) throws IOException + { + RowStats.serializer.serialize(header.stats, out); + + writeType(header.keyType, out); + out.writeShort(header.clusteringTypes.size()); + for (AbstractType type : header.clusteringTypes) + writeType(type, out); + + writeColumnsWithTypes(header.staticColumns, out); + writeColumnsWithTypes(header.regularColumns, out); + } + + // For SSTables + public Component deserialize(Version version, DataInput in) throws IOException + { + RowStats stats = RowStats.serializer.deserialize(in); + + AbstractType keyType = readType(in); + int size = in.readUnsignedShort(); + List> clusteringTypes = new ArrayList<>(size); + for (int i = 0; i < size; i++) + clusteringTypes.add(readType(in)); + + Map> staticColumns = new LinkedHashMap<>(); + Map> regularColumns = new LinkedHashMap<>(); + + readColumnsWithType(in, staticColumns); + readColumnsWithType(in, regularColumns); + + return new Component(keyType, clusteringTypes, staticColumns, regularColumns, stats); + } + + // For SSTables + public int serializedSize(Component header) + { + TypeSizes sizes = TypeSizes.NATIVE; + int size = RowStats.serializer.serializedSize(header.stats, sizes); + + size += sizeofType(header.keyType, sizes); + size += sizes.sizeof((short)header.clusteringTypes.size()); + for (AbstractType type : header.clusteringTypes) + size += sizeofType(type, sizes); + + size += sizeofColumnsWithTypes(header.staticColumns, sizes); + size += sizeofColumnsWithTypes(header.regularColumns, sizes); + return size; + } + + private void writeColumnsWithTypes(Map> columns, DataOutputPlus out) throws IOException + { + out.writeShort(columns.size()); + for (Map.Entry> entry : columns.entrySet()) + { + ByteBufferUtil.writeWithShortLength(entry.getKey(), out); + writeType(entry.getValue(), out); + } + } + + private long sizeofColumnsWithTypes(Map> columns, TypeSizes sizes) + { + long size = sizes.sizeof((short)columns.size()); + for (Map.Entry> entry : columns.entrySet()) + { + size += sizes.sizeofWithShortLength(entry.getKey()); + size += sizeofType(entry.getValue(), sizes); + } + return size; + } + + private void readColumnsWithType(DataInput in, Map> typeMap) throws IOException + { + int length = in.readUnsignedShort(); + for (int i = 0; i < length; i++) + { + ByteBuffer name = ByteBufferUtil.readWithShortLength(in); + typeMap.put(name, readType(in)); + } + } + + private void writeType(AbstractType type, DataOutputPlus out) throws IOException + { + // TODO: we should have a terser serializaion format. Not a big deal though + ByteBufferUtil.writeWithLength(UTF8Type.instance.decompose(type.toString()), out); + } + + private AbstractType readType(DataInput in) throws IOException + { + ByteBuffer raw = ByteBufferUtil.readWithLength(in); + return TypeParser.parse(UTF8Type.instance.compose(raw)); + } + + private int sizeofType(AbstractType type, TypeSizes sizes) + { + return sizes.sizeofWithLength(UTF8Type.instance.decompose(type.toString())); + } + } +} diff --git a/src/java/org/apache/cassandra/db/Serializers.java b/src/java/org/apache/cassandra/db/Serializers.java new file mode 100644 index 0000000000..862d02e95b --- /dev/null +++ b/src/java/org/apache/cassandra/db/Serializers.java @@ -0,0 +1,71 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.cassandra.db; + +import java.io.*; + +import org.apache.cassandra.config.CFMetaData; +import org.apache.cassandra.io.ISerializer; +import org.apache.cassandra.io.util.DataOutputPlus; +import org.apache.cassandra.io.sstable.format.Version; + +import static org.apache.cassandra.io.sstable.IndexHelper.IndexInfo; + +/** + * Holds references on serializers that depend on the table definition. + */ +public class Serializers +{ + private final CFMetaData metadata; + + public Serializers(CFMetaData metadata) + { + this.metadata = metadata; + } + + public IndexInfo.Serializer indexSerializer(Version version) + { + return new IndexInfo.Serializer(metadata, version); + } + + // Note that for the old layout, this will actually discard the cellname parts that are not strictly + // part of the clustering prefix. Don't use this if that's not what you want. + public ISerializer clusteringPrefixSerializer(final Version version, final SerializationHeader header) + { + if (!version.storeRows()) + throw new UnsupportedOperationException(); + + return new ISerializer() + { + public void serialize(ClusteringPrefix clustering, DataOutputPlus out) throws IOException + { + ClusteringPrefix.serializer.serialize(clustering, out, version.correspondingMessagingVersion(), header.clusteringTypes()); + } + + public ClusteringPrefix deserialize(DataInput in) throws IOException + { + return ClusteringPrefix.serializer.deserialize(in, version.correspondingMessagingVersion(), header.clusteringTypes()); + } + + public long serializedSize(ClusteringPrefix clustering, TypeSizes sizes) + { + return ClusteringPrefix.serializer.serializedSize(clustering, version.correspondingMessagingVersion(), header.clusteringTypes(), sizes); + } + }; + } +} diff --git a/src/java/org/apache/cassandra/db/SimpleClustering.java b/src/java/org/apache/cassandra/db/SimpleClustering.java new file mode 100644 index 0000000000..8b1cb7b43a --- /dev/null +++ b/src/java/org/apache/cassandra/db/SimpleClustering.java @@ -0,0 +1,93 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.cassandra.db; + +import java.nio.ByteBuffer; + +import org.apache.cassandra.utils.ObjectSizes; + +public class SimpleClustering extends Clustering +{ + private static final long EMPTY_SIZE = ObjectSizes.measure(new SimpleClustering(new ByteBuffer[0])); + + private final ByteBuffer[] values; + + public SimpleClustering(ByteBuffer... values) + { + this.values = values; + } + + public SimpleClustering(ByteBuffer value) + { + this(new ByteBuffer[]{ value }); + } + + public int size() + { + return values.length; + } + + public ByteBuffer get(int i) + { + return values[i]; + } + + public ByteBuffer[] getRawValues() + { + return values; + } + + @Override + public long unsharedHeapSize() + { + return EMPTY_SIZE + ObjectSizes.sizeOnHeapOf(values); + } + + @Override + public Clustering takeAlias() + { + return this; + } + + public static Builder builder(int size) + { + return new Builder(size); + } + + public static class Builder implements Writer + { + private final ByteBuffer[] values; + private int idx; + + private Builder(int size) + { + this.values = new ByteBuffer[size]; + } + + public void writeClusteringValue(ByteBuffer value) + { + values[idx++] = value; + } + + public SimpleClustering build() + { + assert idx == values.length; + return new SimpleClustering(values); + } + } +} diff --git a/src/java/org/apache/cassandra/db/SimpleDeletionTime.java b/src/java/org/apache/cassandra/db/SimpleDeletionTime.java new file mode 100644 index 0000000000..738c5e6831 --- /dev/null +++ b/src/java/org/apache/cassandra/db/SimpleDeletionTime.java @@ -0,0 +1,61 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.cassandra.db; + +import java.io.DataInput; +import java.io.IOException; + +import com.google.common.annotations.VisibleForTesting; +import com.google.common.base.Objects; + +import org.apache.cassandra.cache.IMeasurableMemory; +import org.apache.cassandra.io.ISerializer; +import org.apache.cassandra.io.util.DataOutputPlus; +import org.apache.cassandra.io.util.FileUtils; +import org.apache.cassandra.utils.ObjectSizes; + +/** + * Simple implementation of DeletionTime. + */ +public class SimpleDeletionTime extends DeletionTime +{ + public final long markedForDeleteAt; + public final int localDeletionTime; + + @VisibleForTesting + public SimpleDeletionTime(long markedForDeleteAt, int localDeletionTime) + { + this.markedForDeleteAt = markedForDeleteAt; + this.localDeletionTime = localDeletionTime; + } + + public long markedForDeleteAt() + { + return markedForDeleteAt; + } + + public int localDeletionTime() + { + return localDeletionTime; + } + + public DeletionTime takeAlias() + { + return this; + } +} diff --git a/src/java/org/apache/cassandra/db/SimpleLivenessInfo.java b/src/java/org/apache/cassandra/db/SimpleLivenessInfo.java new file mode 100644 index 0000000000..fea1b865cd --- /dev/null +++ b/src/java/org/apache/cassandra/db/SimpleLivenessInfo.java @@ -0,0 +1,75 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.cassandra.db; + +import java.util.Objects; +import java.security.MessageDigest; + +import org.apache.cassandra.config.CFMetaData; +import org.apache.cassandra.serializers.MarshalException; +import org.apache.cassandra.utils.FBUtilities; + +public class SimpleLivenessInfo extends AbstractLivenessInfo +{ + private final long timestamp; + private final int ttl; + private final int localDeletionTime; + + // Note that while some code use this ctor, the two following static creation methods + // are usually less error prone. + SimpleLivenessInfo(long timestamp, int ttl, int localDeletionTime) + { + this.timestamp = timestamp; + this.ttl = ttl; + this.localDeletionTime = localDeletionTime; + } + + public static SimpleLivenessInfo forUpdate(long timestamp, int ttl, int nowInSec, CFMetaData metadata) + { + if (ttl == NO_TTL) + ttl = metadata.getDefaultTimeToLive(); + + return new SimpleLivenessInfo(timestamp, ttl, ttl == NO_TTL ? NO_DELETION_TIME : nowInSec + ttl); + } + + public static SimpleLivenessInfo forDeletion(long timestamp, int localDeletionTime) + { + return new SimpleLivenessInfo(timestamp, NO_TTL, localDeletionTime); + } + + public long timestamp() + { + return timestamp; + } + + public int ttl() + { + return ttl; + } + + public int localDeletionTime() + { + return localDeletionTime; + } + + @Override + public LivenessInfo takeAlias() + { + return this; + } +} diff --git a/src/java/org/apache/cassandra/db/SinglePartitionNamesCommand.java b/src/java/org/apache/cassandra/db/SinglePartitionNamesCommand.java new file mode 100644 index 0000000000..d359b2bf26 --- /dev/null +++ b/src/java/org/apache/cassandra/db/SinglePartitionNamesCommand.java @@ -0,0 +1,249 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.cassandra.db; + +import java.util.*; + +import com.google.common.collect.Sets; + +import org.apache.cassandra.concurrent.Stage; +import org.apache.cassandra.concurrent.StageManager; +import org.apache.cassandra.config.CFMetaData; +import org.apache.cassandra.config.ColumnDefinition; +import org.apache.cassandra.db.rows.*; +import org.apache.cassandra.db.partitions.*; +import org.apache.cassandra.db.filter.*; +import org.apache.cassandra.io.sstable.format.SSTableReader; +import org.apache.cassandra.metrics.ColumnFamilyMetrics.Sampler; +import org.apache.cassandra.thrift.ThriftResultsMerger; +import org.apache.cassandra.tracing.Tracing; +import org.apache.cassandra.utils.SearchIterator; +import org.apache.cassandra.utils.memory.HeapAllocator; + +/** + * General interface for storage-engine read queries. + */ +public class SinglePartitionNamesCommand extends SinglePartitionReadCommand +{ + protected SinglePartitionNamesCommand(boolean isDigest, + boolean isForThrift, + CFMetaData metadata, + int nowInSec, + ColumnFilter columnFilter, + RowFilter rowFilter, + DataLimits limits, + DecoratedKey partitionKey, + ClusteringIndexNamesFilter clusteringIndexFilter) + { + super(isDigest, isForThrift, metadata, nowInSec, columnFilter, rowFilter, limits, partitionKey, clusteringIndexFilter); + } + + public SinglePartitionNamesCommand(CFMetaData metadata, + int nowInSec, + ColumnFilter columnFilter, + RowFilter rowFilter, + DataLimits limits, + DecoratedKey partitionKey, + ClusteringIndexNamesFilter clusteringIndexFilter) + { + this(false, false, metadata, nowInSec, columnFilter, rowFilter, limits, partitionKey, clusteringIndexFilter); + } + + public SinglePartitionNamesCommand copy() + { + return new SinglePartitionNamesCommand(isDigestQuery(), isForThrift(), metadata(), nowInSec(), columnFilter(), rowFilter(), limits(), partitionKey(), clusteringIndexFilter()); + } + + protected UnfilteredRowIterator queryMemtableAndDiskInternal(ColumnFamilyStore cfs, boolean copyOnHeap) + { + Tracing.trace("Acquiring sstable references"); + ColumnFamilyStore.ViewFragment view = cfs.select(cfs.viewFilter(partitionKey())); + + ArrayBackedPartition result = null; + ClusteringIndexNamesFilter filter = clusteringIndexFilter(); + + Tracing.trace("Merging memtable contents"); + for (Memtable memtable : view.memtables) + { + Partition partition = memtable.getPartition(partitionKey()); + if (partition == null) + continue; + + try (UnfilteredRowIterator iter = filter.getUnfilteredRowIterator(columnFilter(), partition)) + { + if (iter.isEmpty()) + continue; + + UnfilteredRowIterator clonedFilter = copyOnHeap + ? UnfilteredRowIterators.cloningIterator(iter, HeapAllocator.instance) + : iter; + result = add(isForThrift() ? ThriftResultsMerger.maybeWrap(clonedFilter, nowInSec()) : clonedFilter, result); + } + } + + /* add the SSTables on disk */ + Collections.sort(view.sstables, SSTableReader.maxTimestampComparator); + int sstablesIterated = 0; + + // read sorted sstables + for (SSTableReader sstable : view.sstables) + { + // if we've already seen a partition tombstone with a timestamp greater + // than the most recent update to this sstable, we're done, since the rest of the sstables + // will also be older + if (result != null && sstable.getMaxTimestamp() < result.partitionLevelDeletion().markedForDeleteAt()) + break; + + long currentMaxTs = sstable.getMaxTimestamp(); + filter = reduceFilter(filter, result, currentMaxTs); + if (filter == null) + break; + + Tracing.trace("Merging data from sstable {}", sstable.descriptor.generation); + sstable.incrementReadCount(); + try (UnfilteredRowIterator iter = filter.filter(sstable.iterator(partitionKey(), columnFilter(), filter.isReversed(), isForThrift()))) + { + if (iter.isEmpty()) + continue; + + sstablesIterated++; + result = add(isForThrift() ? ThriftResultsMerger.maybeWrap(iter, nowInSec()) : iter, result); + } + } + + cfs.metric.updateSSTableIterated(sstablesIterated); + + if (result == null || result.isEmpty()) + return UnfilteredRowIterators.emptyIterator(metadata(), partitionKey(), false); + + DecoratedKey key = result.partitionKey(); + cfs.metric.samplers.get(Sampler.READS).addSample(key.getKey(), key.hashCode(), 1); + + // "hoist up" the requested data into a more recent sstable + if (sstablesIterated > cfs.getMinimumCompactionThreshold() + && !cfs.isAutoCompactionDisabled() + && cfs.getCompactionStrategyManager().shouldDefragment()) + { + // !!WARNING!! if we stop copying our data to a heap-managed object, + // we will need to track the lifetime of this mutation as well + Tracing.trace("Defragmenting requested data"); + + try (UnfilteredRowIterator iter = result.unfilteredIterator(columnFilter(), Slices.ALL, false)) + { + final Mutation mutation = new Mutation(UnfilteredRowIterators.toUpdate(iter)); + StageManager.getStage(Stage.MUTATION).execute(new Runnable() + { + public void run() + { + // skipping commitlog and index updates is fine since we're just de-fragmenting existing data + Keyspace.open(mutation.getKeyspaceName()).apply(mutation, false, false); + } + }); + } + } + + return result.unfilteredIterator(columnFilter(), Slices.ALL, clusteringIndexFilter().isReversed()); + } + + private ArrayBackedPartition add(UnfilteredRowIterator iter, ArrayBackedPartition result) + { + int maxRows = Math.max(clusteringIndexFilter().requestedRows().size(), 1); + if (result == null) + return ArrayBackedPartition.create(iter, maxRows); + + try (UnfilteredRowIterator merged = UnfilteredRowIterators.merge(Arrays.asList(iter, result.unfilteredIterator(columnFilter(), Slices.ALL, false)), nowInSec())) + { + return ArrayBackedPartition.create(merged, maxRows); + } + } + + private ClusteringIndexNamesFilter reduceFilter(ClusteringIndexNamesFilter filter, Partition result, long sstableTimestamp) + { + if (result == null) + return filter; + + SearchIterator searchIter = result.searchIterator(columnFilter(), false); + + PartitionColumns columns = columnFilter().fetchedColumns(); + NavigableSet clusterings = filter.requestedRows(); + + // We want to remove rows for which we have values for all requested columns. We have to deal with both static and regular rows. + // TODO: we could also remove a selected column if we've found values for every requested row but we'll leave + // that for later. + + boolean removeStatic = false; + if (!columns.statics.isEmpty()) + { + Row staticRow = searchIter.next(Clustering.STATIC_CLUSTERING); + removeStatic = staticRow != null && canRemoveRow(staticRow, columns.statics, sstableTimestamp); + } + + NavigableSet toRemove = null; + for (Clustering clustering : clusterings) + { + if (!searchIter.hasNext()) + break; + + Row row = searchIter.next(clustering); + if (row == null || !canRemoveRow(row, columns.regulars, sstableTimestamp)) + continue; + + if (toRemove == null) + toRemove = new TreeSet<>(result.metadata().comparator); + toRemove.add(clustering); + } + + if (!removeStatic && toRemove == null) + return filter; + + // Check if we have everything we need + boolean hasNoMoreStatic = columns.statics.isEmpty() || removeStatic; + boolean hasNoMoreClusterings = clusterings.isEmpty() || (toRemove != null && toRemove.size() == clusterings.size()); + if (hasNoMoreStatic && hasNoMoreClusterings) + return null; + + NavigableSet newClusterings = clusterings; + if (toRemove != null) + { + newClusterings = new TreeSet<>(result.metadata().comparator); + newClusterings.addAll(Sets.difference(clusterings, toRemove)); + } + return new ClusteringIndexNamesFilter(newClusterings, filter.isReversed()); + } + + private boolean canRemoveRow(Row row, Columns requestedColumns, long sstableTimestamp) + { + // We can remove a row if it has data that is more recent that the next sstable to consider for the data that the query + // cares about. And the data we care about is 1) the row timestamp (since every query cares if the row exists or not) + // and 2) the requested columns. + if (!row.primaryKeyLivenessInfo().hasTimestamp() || row.primaryKeyLivenessInfo().timestamp() <= sstableTimestamp) + return false; + + for (ColumnDefinition column : requestedColumns) + { + // We can never be sure we have all of a collection, so never remove rows in that case. + if (column.type.isCollection()) + return false; + + Cell cell = row.getCell(column); + if (cell == null || cell.livenessInfo().timestamp() <= sstableTimestamp) + return false; + } + return true; + } +} diff --git a/src/java/org/apache/cassandra/db/SinglePartitionReadCommand.java b/src/java/org/apache/cassandra/db/SinglePartitionReadCommand.java new file mode 100644 index 0000000000..38651c1686 --- /dev/null +++ b/src/java/org/apache/cassandra/db/SinglePartitionReadCommand.java @@ -0,0 +1,498 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.cassandra.db; + +import java.io.DataInput; +import java.io.IOException; +import java.util.*; + +import org.apache.cassandra.cache.*; +import org.apache.cassandra.config.CFMetaData; +import org.apache.cassandra.config.ColumnDefinition; +import org.apache.cassandra.config.DatabaseDescriptor; +import org.apache.cassandra.db.rows.*; +import org.apache.cassandra.db.filter.*; +import org.apache.cassandra.db.partitions.*; +import org.apache.cassandra.exceptions.RequestExecutionException; +import org.apache.cassandra.io.util.DataOutputPlus; +import org.apache.cassandra.metrics.ColumnFamilyMetrics; +import org.apache.cassandra.service.*; +import org.apache.cassandra.service.pager.*; +import org.apache.cassandra.tracing.Tracing; +import org.apache.cassandra.utils.concurrent.OpOrder; + +/** + * A read command that selects a (part of a) single partition. + */ +public abstract class SinglePartitionReadCommand extends ReadCommand +{ + protected static final SelectionDeserializer selectionDeserializer = new Deserializer(); + + private final DecoratedKey partitionKey; + private final F clusteringIndexFilter; + + protected SinglePartitionReadCommand(boolean isDigest, + boolean isForThrift, + CFMetaData metadata, + int nowInSec, + ColumnFilter columnFilter, + RowFilter rowFilter, + DataLimits limits, + DecoratedKey partitionKey, + F clusteringIndexFilter) + { + super(Kind.SINGLE_PARTITION, isDigest, isForThrift, metadata, nowInSec, columnFilter, rowFilter, limits); + this.partitionKey = partitionKey; + this.clusteringIndexFilter = clusteringIndexFilter; + } + + /** + * Creates a new read command on a single partition. + * + * @param metadata the table to query. + * @param nowInSec the time in seconds to use are "now" for this query. + * @param columnFilter the column filter to use for the query. + * @param rowFilter the row filter to use for the query. + * @param limits the limits to use for the query. + * @param partitionKey the partition key for the partition to query. + * @param clusteringIndexFilter the clustering index filter to use for the query. + * + * @return a newly created read command. + */ + public static SinglePartitionReadCommand create(CFMetaData metadata, + int nowInSec, + ColumnFilter columnFilter, + RowFilter rowFilter, + DataLimits limits, + DecoratedKey partitionKey, + ClusteringIndexFilter clusteringIndexFilter) + { + return create(false, metadata, nowInSec, columnFilter, rowFilter, limits, partitionKey, clusteringIndexFilter); + } + + /** + * Creates a new read command on a single partition for thrift. + * + * @param isForThrift whether the query is for thrift or not. + * @param metadata the table to query. + * @param nowInSec the time in seconds to use are "now" for this query. + * @param columnFilter the column filter to use for the query. + * @param rowFilter the row filter to use for the query. + * @param limits the limits to use for the query. + * @param partitionKey the partition key for the partition to query. + * @param clusteringIndexFilter the clustering index filter to use for the query. + * + * @return a newly created read command. + */ + public static SinglePartitionReadCommand create(boolean isForThrift, + CFMetaData metadata, + int nowInSec, + ColumnFilter columnFilter, + RowFilter rowFilter, + DataLimits limits, + DecoratedKey partitionKey, + ClusteringIndexFilter clusteringIndexFilter) + { + if (clusteringIndexFilter instanceof ClusteringIndexSliceFilter) + return new SinglePartitionSliceCommand(false, isForThrift, metadata, nowInSec, columnFilter, rowFilter, limits, partitionKey, (ClusteringIndexSliceFilter) clusteringIndexFilter); + + assert clusteringIndexFilter instanceof ClusteringIndexNamesFilter; + return new SinglePartitionNamesCommand(false, isForThrift, metadata, nowInSec, columnFilter, rowFilter, limits, partitionKey, (ClusteringIndexNamesFilter) clusteringIndexFilter); + } + + /** + * Creates a new read command on a single partition. + * + * @param metadata the table to query. + * @param nowInSec the time in seconds to use are "now" for this query. + * @param key the partition key for the partition to query. + * @param columnFilter the column filter to use for the query. + * @param filter the clustering index filter to use for the query. + * + * @return a newly created read command. The returned command will use no row filter and have no limits. + */ + public static SinglePartitionReadCommand create(CFMetaData metadata, int nowInSec, DecoratedKey key, ColumnFilter columnFilter, ClusteringIndexFilter filter) + { + return create(metadata, nowInSec, columnFilter, RowFilter.NONE, DataLimits.NONE, key, filter); + } + + /** + * Creates a new read command that queries a single partition in its entirety. + * + * @param metadata the table to query. + * @param nowInSec the time in seconds to use are "now" for this query. + * @param key the partition key for the partition to query. + * + * @return a newly created read command that queries all the rows of {@code key}. + */ + public static SinglePartitionReadCommand fullPartitionRead(CFMetaData metadata, int nowInSec, DecoratedKey key) + { + return SinglePartitionSliceCommand.create(metadata, nowInSec, key, Slices.ALL); + } + + public DecoratedKey partitionKey() + { + return partitionKey; + } + + public F clusteringIndexFilter() + { + return clusteringIndexFilter; + } + + public ClusteringIndexFilter clusteringIndexFilter(DecoratedKey key) + { + return clusteringIndexFilter; + } + + public long getTimeout() + { + return DatabaseDescriptor.getReadRpcTimeout(); + } + + public boolean selects(DecoratedKey partitionKey, Clustering clustering) + { + if (!partitionKey().equals(partitionKey)) + return false; + + if (clustering == Clustering.STATIC_CLUSTERING) + return !columnFilter().fetchedColumns().statics.isEmpty(); + + return clusteringIndexFilter().selects(clustering); + } + + /** + * Returns a new command suitable to paging from the last returned row. + * + * @param lastReturned the last row returned by the previous page. The newly created command + * will only query row that comes after this (in query order). This can be {@code null} if this + * is the first page. + * @param pageSize the size to use for the page to query. + * + * @return the newly create command. + */ + public SinglePartitionReadCommand forPaging(Clustering lastReturned, int pageSize) + { + // We shouldn't have set digest yet when reaching that point + assert !isDigestQuery(); + return create(isForThrift(), + metadata(), + nowInSec(), + columnFilter(), + rowFilter(), + limits().forPaging(pageSize), + partitionKey(), + lastReturned == null ? clusteringIndexFilter() : clusteringIndexFilter.forPaging(metadata().comparator, lastReturned, false)); + } + + public PartitionIterator execute(ConsistencyLevel consistency, ClientState clientState) throws RequestExecutionException + { + return StorageProxy.read(Group.one(this), consistency, clientState); + } + + public SinglePartitionPager getPager(PagingState pagingState) + { + return getPager(this, pagingState); + } + + private static SinglePartitionPager getPager(SinglePartitionReadCommand command, PagingState pagingState) + { + return new SinglePartitionPager(command, pagingState); + } + + protected void recordLatency(ColumnFamilyMetrics metric, long latencyNanos) + { + metric.readLatency.addNano(latencyNanos); + } + + protected UnfilteredPartitionIterator queryStorage(final ColumnFamilyStore cfs, ReadOrderGroup orderGroup) + { + @SuppressWarnings("resource") // we close the created iterator through closing the result of this method (and SingletonUnfilteredPartitionIterator ctor cannot fail) + UnfilteredRowIterator partition = cfs.isRowCacheEnabled() + ? getThroughCache(cfs, orderGroup.baseReadOpOrderGroup()) + : queryMemtableAndDisk(cfs, orderGroup.baseReadOpOrderGroup()); + return new SingletonUnfilteredPartitionIterator(partition, isForThrift()); + } + + /** + * Fetch the rows requested if in cache; if not, read it from disk and cache it. + *

+ * If the partition is cached, and the filter given is within its bounds, we return + * from cache, otherwise from disk. + *

+ * If the partition is is not cached, we figure out what filter is "biggest", read + * that from disk, then filter the result and either cache that or return it. + */ + private UnfilteredRowIterator getThroughCache(ColumnFamilyStore cfs, OpOrder.Group readOp) + { + assert !cfs.isIndex(); // CASSANDRA-5732 + assert cfs.isRowCacheEnabled() : String.format("Row cache is not enabled on table [" + cfs.name + "]"); + + UUID cfId = metadata().cfId; + RowCacheKey key = new RowCacheKey(cfId, partitionKey()); + + // Attempt a sentinel-read-cache sequence. if a write invalidates our sentinel, we'll return our + // (now potentially obsolete) data, but won't cache it. see CASSANDRA-3862 + // TODO: don't evict entire partitions on writes (#2864) + IRowCacheEntry cached = CacheService.instance.rowCache.get(key); + if (cached != null) + { + if (cached instanceof RowCacheSentinel) + { + // Some other read is trying to cache the value, just do a normal non-caching read + Tracing.trace("Row cache miss (race)"); + cfs.metric.rowCacheMiss.inc(); + return queryMemtableAndDisk(cfs, readOp); + } + + CachedPartition cachedPartition = (CachedPartition)cached; + if (cfs.isFilterFullyCoveredBy(clusteringIndexFilter(), limits(), cachedPartition, nowInSec())) + { + cfs.metric.rowCacheHit.inc(); + Tracing.trace("Row cache hit"); + return clusteringIndexFilter().getUnfilteredRowIterator(columnFilter(), cachedPartition); + } + + cfs.metric.rowCacheHitOutOfRange.inc(); + Tracing.trace("Ignoring row cache as cached value could not satisfy query"); + return queryMemtableAndDisk(cfs, readOp); + } + + cfs.metric.rowCacheMiss.inc(); + Tracing.trace("Row cache miss"); + + boolean cacheFullPartitions = metadata().getCaching().rowCache.cacheFullPartitions(); + + // To be able to cache what we read, what we read must at least covers what the cache holds, that + // is the 'rowsToCache' first rows of the partition. We could read those 'rowsToCache' first rows + // systematically, but we'd have to "extend" that to whatever is needed for the user query that the + // 'rowsToCache' first rows don't cover and it's not trivial with our existing filters. So currently + // we settle for caching what we read only if the user query does query the head of the partition since + // that's the common case of when we'll be able to use the cache anyway. One exception is if we cache + // full partitions, in which case we just always read it all and cache. + if (cacheFullPartitions || clusteringIndexFilter().isHeadFilter()) + { + RowCacheSentinel sentinel = new RowCacheSentinel(); + boolean sentinelSuccess = CacheService.instance.rowCache.putIfAbsent(key, sentinel); + boolean sentinelReplaced = false; + + try + { + int rowsToCache = cacheFullPartitions ? Integer.MAX_VALUE : metadata().getCaching().rowCache.rowsToCache; + @SuppressWarnings("resource") // we close on exception or upon closing the result of this method + UnfilteredRowIterator iter = SinglePartitionReadCommand.fullPartitionRead(metadata(), nowInSec(), partitionKey()).queryMemtableAndDisk(cfs, readOp); + try + { + // We want to cache only rowsToCache rows + CachedPartition toCache = ArrayBackedCachedPartition.create(DataLimits.cqlLimits(rowsToCache).filter(iter, nowInSec()), nowInSec()); + if (sentinelSuccess && !toCache.isEmpty()) + { + Tracing.trace("Caching {} rows", toCache.rowCount()); + CacheService.instance.rowCache.replace(key, sentinel, toCache); + // Whether or not the previous replace has worked, our sentinel is not in the cache anymore + sentinelReplaced = true; + } + + // We then re-filter out what this query wants. + // Note that in the case where we don't cache full partitions, it's possible that the current query is interested in more + // than what we've cached, so we can't just use toCache. + UnfilteredRowIterator cacheIterator = clusteringIndexFilter().getUnfilteredRowIterator(columnFilter(), toCache); + if (cacheFullPartitions) + { + // Everything is guaranteed to be in 'toCache', we're done with 'iter' + assert !iter.hasNext(); + iter.close(); + return cacheIterator; + } + return UnfilteredRowIterators.concat(cacheIterator, clusteringIndexFilter().filterNotIndexed(columnFilter(), iter)); + } + catch (RuntimeException | Error e) + { + iter.close(); + throw e; + } + } + finally + { + if (sentinelSuccess && !sentinelReplaced) + cfs.invalidateCachedPartition(key); + } + } + + Tracing.trace("Fetching data but not populating cache as query does not query from the start of the partition"); + return queryMemtableAndDisk(cfs, readOp); + } + + /** + * Queries both memtable and sstables to fetch the result of this query. + *

+ * Please note that this method: + * 1) does not check the row cache. + * 2) does not apply the query limit, nor the row filter (and so ignore 2ndary indexes). + * Those are applied in {@link ReadCommand#executeLocally}. + * 3) does not record some of the read metrics (latency, scanned cells histograms) nor + * throws TombstoneOverwhelmingException. + * It is publicly exposed because there is a few places where that is exactly what we want, + * but it should be used only where you know you don't need thoses things. + *

+ * Also note that one must have "started" a {@code OpOrder.Group} on the queried table, and that is + * to enforce that that it is required as parameter, even though it's not explicitlly used by the method. + */ + public UnfilteredRowIterator queryMemtableAndDisk(ColumnFamilyStore cfs, OpOrder.Group readOp) + { + Tracing.trace("Executing single-partition query on {}", cfs.name); + + boolean copyOnHeap = Memtable.MEMORY_POOL.needToCopyOnHeap(); + return queryMemtableAndDiskInternal(cfs, copyOnHeap); + } + + protected abstract UnfilteredRowIterator queryMemtableAndDiskInternal(ColumnFamilyStore cfs, boolean copyOnHeap); + + @Override + public String toString() + { + return String.format("Read(%s.%s columns=%s rowFilter=%s limits=%s key=%s filter=%s, nowInSec=%d)", + metadata().ksName, + metadata().cfName, + columnFilter(), + rowFilter(), + limits(), + metadata().getKeyValidator().getString(partitionKey().getKey()), + clusteringIndexFilter.toString(metadata()), + nowInSec()); + } + + protected void appendCQLWhereClause(StringBuilder sb) + { + sb.append(" WHERE "); + + sb.append(ColumnDefinition.toCQLString(metadata().partitionKeyColumns())).append(" = "); + DataRange.appendKeyString(sb, metadata().getKeyValidator(), partitionKey().getKey()); + + // We put the row filter first because the clustering index filter can end by "ORDER BY" + if (!rowFilter().isEmpty()) + sb.append(" AND ").append(rowFilter()); + + String filterString = clusteringIndexFilter().toCQLString(metadata()); + if (!filterString.isEmpty()) + sb.append(" AND ").append(filterString); + } + + protected void serializeSelection(DataOutputPlus out, int version) throws IOException + { + metadata().getKeyValidator().writeValue(partitionKey().getKey(), out); + ClusteringIndexFilter.serializer.serialize(clusteringIndexFilter(), out, version); + } + + protected long selectionSerializedSize(int version) + { + TypeSizes sizes = TypeSizes.NATIVE; + return metadata().getKeyValidator().writtenLength(partitionKey().getKey(), sizes) + + ClusteringIndexFilter.serializer.serializedSize(clusteringIndexFilter(), version); + } + + /** + * Groups multiple single partition read commands. + */ + public static class Group implements ReadQuery + { + public final List> commands; + private final DataLimits limits; + private final int nowInSec; + + public Group(List> commands, DataLimits limits) + { + assert !commands.isEmpty(); + this.commands = commands; + this.limits = limits; + this.nowInSec = commands.get(0).nowInSec(); + for (int i = 1; i < commands.size(); i++) + assert commands.get(i).nowInSec() == nowInSec; + } + + public static Group one(SinglePartitionReadCommand command) + { + return new Group(Collections.>singletonList(command), command.limits()); + } + + public PartitionIterator execute(ConsistencyLevel consistency, ClientState clientState) throws RequestExecutionException + { + return StorageProxy.read(this, consistency, clientState); + } + + public int nowInSec() + { + return nowInSec; + } + + public DataLimits limits() + { + return limits; + } + + public CFMetaData metadata() + { + return commands.get(0).metadata(); + } + + public ReadOrderGroup startOrderGroup() + { + // Note that the only difference between the command in a group must be the partition key on which + // they applied. So as far as ReadOrderGroup is concerned, we can use any of the commands to start one. + return commands.get(0).startOrderGroup(); + } + + public PartitionIterator executeInternal(ReadOrderGroup orderGroup) + { + List partitions = new ArrayList<>(commands.size()); + for (SinglePartitionReadCommand cmd : commands) + partitions.add(cmd.executeInternal(orderGroup)); + + // Because we only have enforce the limit per command, we need to enforce it globally. + return limits.filter(PartitionIterators.concat(partitions), nowInSec); + } + + public QueryPager getPager(PagingState pagingState) + { + if (commands.size() == 1) + return SinglePartitionReadCommand.getPager(commands.get(0), pagingState); + + return new MultiPartitionPager(this, pagingState); + } + + @Override + public String toString() + { + return commands.toString(); + } + } + + private static class Deserializer extends SelectionDeserializer + { + public ReadCommand deserialize(DataInput in, int version, boolean isDigest, boolean isForThrift, CFMetaData metadata, int nowInSec, ColumnFilter columnFilter, RowFilter rowFilter, DataLimits limits) + throws IOException + { + DecoratedKey key = StorageService.getPartitioner().decorateKey(metadata.getKeyValidator().readValue(in)); + ClusteringIndexFilter filter = ClusteringIndexFilter.serializer.deserialize(in, version, metadata); + if (filter instanceof ClusteringIndexNamesFilter) + return new SinglePartitionNamesCommand(isDigest, isForThrift, metadata, nowInSec, columnFilter, rowFilter, limits, key, (ClusteringIndexNamesFilter)filter); + else + return new SinglePartitionSliceCommand(isDigest, isForThrift, metadata, nowInSec, columnFilter, rowFilter, limits, key, (ClusteringIndexSliceFilter)filter); + } + }; +} diff --git a/src/java/org/apache/cassandra/db/SinglePartitionSliceCommand.java b/src/java/org/apache/cassandra/db/SinglePartitionSliceCommand.java new file mode 100644 index 0000000000..65b4e3f4fc --- /dev/null +++ b/src/java/org/apache/cassandra/db/SinglePartitionSliceCommand.java @@ -0,0 +1,232 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.cassandra.db; + +import java.util.*; + +import com.google.common.collect.Iterables; + +import org.apache.cassandra.config.CFMetaData; +import org.apache.cassandra.db.filter.ClusteringIndexSliceFilter; +import org.apache.cassandra.db.rows.*; +import org.apache.cassandra.db.filter.*; +import org.apache.cassandra.db.partitions.Partition; +import org.apache.cassandra.io.sstable.format.SSTableReader; +import org.apache.cassandra.metrics.ColumnFamilyMetrics.Sampler; +import org.apache.cassandra.thrift.ThriftResultsMerger; +import org.apache.cassandra.tracing.Tracing; +import org.apache.cassandra.utils.FBUtilities; +import org.apache.cassandra.utils.memory.HeapAllocator; + +/** + * General interface for storage-engine read queries. + */ +public class SinglePartitionSliceCommand extends SinglePartitionReadCommand +{ + public SinglePartitionSliceCommand(boolean isDigest, + boolean isForThrift, + CFMetaData metadata, + int nowInSec, + ColumnFilter columnFilter, + RowFilter rowFilter, + DataLimits limits, + DecoratedKey partitionKey, + ClusteringIndexSliceFilter clusteringIndexFilter) + { + super(isDigest, isForThrift, metadata, nowInSec, columnFilter, rowFilter, limits, partitionKey, clusteringIndexFilter); + } + + public SinglePartitionSliceCommand(CFMetaData metadata, + int nowInSec, + ColumnFilter columnFilter, + RowFilter rowFilter, + DataLimits limits, + DecoratedKey partitionKey, + ClusteringIndexSliceFilter clusteringIndexFilter) + { + this(false, false, metadata, nowInSec, columnFilter, rowFilter, limits, partitionKey, clusteringIndexFilter); + } + + /** + * Creates a new single partition slice command for the provided single slice. + * + * @param metadata the table to query. + * @param nowInSec the time in seconds to use are "now" for this query. + * @param key the partition key for the partition to query. + * @param slice the slice of rows to query. + * + * @return a newly created read command that queries {@code slice} in {@code key}. The returned query will + * query every columns for the table (without limit or row filtering) and be in forward order. + */ + public static SinglePartitionReadCommand create(CFMetaData metadata, int nowInSec, DecoratedKey key, Slice slice) + { + return create(metadata, nowInSec, key, Slices.with(metadata.comparator, slice)); + } + + /** + * Creates a new single partition slice command for the provided slices. + * + * @param metadata the table to query. + * @param nowInSec the time in seconds to use are "now" for this query. + * @param key the partition key for the partition to query. + * @param slices the slices of rows to query. + * + * @return a newly created read command that queries the {@code slices} in {@code key}. The returned query will + * query every columns for the table (without limit or row filtering) and be in forward order. + */ + public static SinglePartitionReadCommand create(CFMetaData metadata, int nowInSec, DecoratedKey key, Slices slices) + { + ClusteringIndexSliceFilter filter = new ClusteringIndexSliceFilter(slices, false); + return new SinglePartitionSliceCommand(metadata, nowInSec, ColumnFilter.all(metadata), RowFilter.NONE, DataLimits.NONE, key, filter); + } + + public SinglePartitionSliceCommand copy() + { + return new SinglePartitionSliceCommand(isDigestQuery(), isForThrift(), metadata(), nowInSec(), columnFilter(), rowFilter(), limits(), partitionKey(), clusteringIndexFilter()); + } + + protected UnfilteredRowIterator queryMemtableAndDiskInternal(ColumnFamilyStore cfs, boolean copyOnHeap) + { + Tracing.trace("Acquiring sstable references"); + ColumnFamilyStore.ViewFragment view = cfs.select(cfs.viewFilter(partitionKey())); + + List iterators = new ArrayList<>(Iterables.size(view.memtables) + view.sstables.size()); + ClusteringIndexSliceFilter filter = clusteringIndexFilter(); + + try + { + for (Memtable memtable : view.memtables) + { + Partition partition = memtable.getPartition(partitionKey()); + if (partition == null) + continue; + + @SuppressWarnings("resource") // 'iter' is added to iterators which is closed on exception, or through the closing of the final merged iterator + UnfilteredRowIterator iter = filter.getUnfilteredRowIterator(columnFilter(), partition); + @SuppressWarnings("resource") // same as above + UnfilteredRowIterator maybeCopied = copyOnHeap ? UnfilteredRowIterators.cloningIterator(iter, HeapAllocator.instance) : iter; + iterators.add(isForThrift() ? ThriftResultsMerger.maybeWrap(maybeCopied, nowInSec()) : maybeCopied); + } + + /* + * We can't eliminate full sstables based on the timestamp of what we've already read like + * in collectTimeOrderedData, but we still want to eliminate sstable whose maxTimestamp < mostRecentTombstone + * we've read. We still rely on the sstable ordering by maxTimestamp since if + * maxTimestamp_s1 > maxTimestamp_s0, + * we're guaranteed that s1 cannot have a row tombstone such that + * timestamp(tombstone) > maxTimestamp_s0 + * since we necessarily have + * timestamp(tombstone) <= maxTimestamp_s1 + * In other words, iterating in maxTimestamp order allow to do our mostRecentPartitionTombstone elimination + * in one pass, and minimize the number of sstables for which we read a partition tombstone. + */ + int sstablesIterated = 0; + Collections.sort(view.sstables, SSTableReader.maxTimestampComparator); + List skippedSSTables = null; + long mostRecentPartitionTombstone = Long.MIN_VALUE; + long minTimestamp = Long.MAX_VALUE; + int nonIntersectingSSTables = 0; + + for (SSTableReader sstable : view.sstables) + { + minTimestamp = Math.min(minTimestamp, sstable.getMinTimestamp()); + // if we've already seen a partition tombstone with a timestamp greater + // than the most recent update to this sstable, we can skip it + if (sstable.getMaxTimestamp() < mostRecentPartitionTombstone) + break; + + if (!filter.shouldInclude(sstable)) + { + nonIntersectingSSTables++; + // sstable contains no tombstone if maxLocalDeletionTime == Integer.MAX_VALUE, so we can safely skip those entirely + if (sstable.getSSTableMetadata().maxLocalDeletionTime != Integer.MAX_VALUE) + { + if (skippedSSTables == null) + skippedSSTables = new ArrayList<>(); + skippedSSTables.add(sstable); + } + continue; + } + + sstable.incrementReadCount(); + @SuppressWarnings("resource") // 'iter' is added to iterators which is closed on exception, or through the closing of the final merged iterator + UnfilteredRowIterator iter = filter.filter(sstable.iterator(partitionKey(), columnFilter(), filter.isReversed(), isForThrift())); + iterators.add(isForThrift() ? ThriftResultsMerger.maybeWrap(iter, nowInSec()) : iter); + mostRecentPartitionTombstone = Math.max(mostRecentPartitionTombstone, iter.partitionLevelDeletion().markedForDeleteAt()); + sstablesIterated++; + } + + int includedDueToTombstones = 0; + // Check for partition tombstones in the skipped sstables + if (skippedSSTables != null) + { + for (SSTableReader sstable : skippedSSTables) + { + if (sstable.getMaxTimestamp() <= minTimestamp) + continue; + + sstable.incrementReadCount(); + @SuppressWarnings("resource") // 'iter' is either closed right away, or added to iterators which is close on exception, or through the closing of the final merged iterator + UnfilteredRowIterator iter = filter.filter(sstable.iterator(partitionKey(), columnFilter(), filter.isReversed(), isForThrift())); + if (iter.partitionLevelDeletion().markedForDeleteAt() > minTimestamp) + { + iterators.add(iter); + includedDueToTombstones++; + sstablesIterated++; + } + else + { + iter.close(); + } + } + } + if (Tracing.isTracing()) + Tracing.trace("Skipped {}/{} non-slice-intersecting sstables, included {} due to tombstones", + nonIntersectingSSTables, view.sstables.size(), includedDueToTombstones); + + cfs.metric.updateSSTableIterated(sstablesIterated); + + if (iterators.isEmpty()) + return UnfilteredRowIterators.emptyIterator(cfs.metadata, partitionKey(), filter.isReversed()); + + Tracing.trace("Merging data from memtables and {} sstables", sstablesIterated); + + @SuppressWarnings("resource") // Closed through the closing of the result of that method. + UnfilteredRowIterator merged = UnfilteredRowIterators.merge(iterators, nowInSec()); + if (!merged.isEmpty()) + { + DecoratedKey key = merged.partitionKey(); + cfs.metric.samplers.get(Sampler.READS).addSample(key.getKey(), key.hashCode(), 1); + } + + return merged; + } + catch (RuntimeException | Error e) + { + try + { + FBUtilities.closeAll(iterators); + } + catch (Exception suppressed) + { + e.addSuppressed(suppressed); + } + throw e; + } + } +} diff --git a/src/java/org/apache/cassandra/db/Slice.java b/src/java/org/apache/cassandra/db/Slice.java new file mode 100644 index 0000000000..dae491e6c2 --- /dev/null +++ b/src/java/org/apache/cassandra/db/Slice.java @@ -0,0 +1,652 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.cassandra.db; + +import java.io.DataInput; +import java.io.IOException; +import java.nio.ByteBuffer; +import java.security.MessageDigest; +import java.util.*; + +import org.apache.cassandra.config.CFMetaData; +import org.apache.cassandra.db.marshal.AbstractType; +import org.apache.cassandra.io.util.DataOutputPlus; +import org.apache.cassandra.utils.FBUtilities; +import org.apache.cassandra.utils.ObjectSizes; + +/** + * A slice represents the selection of a range of rows. + *

+ * A slice has a start and an end bound that are both (potentially full) clustering prefixes. + * A slice selects every rows whose clustering is bigger than the slice start prefix but smaller + * than the end prefix. Both start and end can be either inclusive or exclusive. + */ +public class Slice +{ + public static final Serializer serializer = new Serializer(); + + /** The slice selecting all rows (of a given partition) */ + public static final Slice ALL = new Slice(Bound.BOTTOM, Bound.TOP) + { + @Override + public boolean selects(ClusteringComparator comparator, Clustering clustering) + { + return true; + } + + @Override + public boolean intersects(ClusteringComparator comparator, List minClusteringValues, List maxClusteringValues) + { + return true; + } + + @Override + public String toString(ClusteringComparator comparator) + { + return "ALL"; + } + }; + + private final Bound start; + private final Bound end; + + private Slice(Bound start, Bound end) + { + assert start.isStart() && end.isEnd(); + this.start = start.takeAlias(); + this.end = end.takeAlias(); + } + + public static Slice make(Bound start, Bound end) + { + if (start == Bound.BOTTOM && end == Bound.TOP) + return ALL; + + return new Slice(start, end); + } + + public static Slice make(ClusteringComparator comparator, Object... values) + { + CBuilder builder = CBuilder.create(comparator); + for (int i = 0; i < values.length; i++) + { + Object val = values[i]; + if (val instanceof ByteBuffer) + builder.add((ByteBuffer)val); + else + builder.add(val); + } + return new Slice(builder.buildBound(true, true), builder.buildBound(false, true)); + } + + public static Slice make(Clustering clustering) + { + // This doesn't give us what we want with the clustering prefix + assert clustering != Clustering.STATIC_CLUSTERING; + ByteBuffer[] values = extractValues(clustering); + return new Slice(Bound.inclusiveStartOf(values), Bound.inclusiveEndOf(values)); + } + + public static Slice make(Clustering start, Clustering end) + { + // This doesn't give us what we want with the clustering prefix + assert start != Clustering.STATIC_CLUSTERING && end != Clustering.STATIC_CLUSTERING; + + ByteBuffer[] startValues = extractValues(start); + ByteBuffer[] endValues = extractValues(end); + + return new Slice(Bound.inclusiveStartOf(startValues), Bound.inclusiveEndOf(endValues)); + } + + private static ByteBuffer[] extractValues(ClusteringPrefix clustering) + { + ByteBuffer[] values = new ByteBuffer[clustering.size()]; + for (int i = 0; i < clustering.size(); i++) + values[i] = clustering.get(i); + return values; + } + + public Bound start() + { + return start; + } + + public Bound end() + { + return end; + } + + public Bound open(boolean reversed) + { + return reversed ? end : start; + } + + public Bound close(boolean reversed) + { + return reversed ? start : end; + } + + /** + * Return whether the slice is empty. + * + * @param comparator the comparator to compare the bounds. + * @return whether the slice formed is empty or not. + */ + public boolean isEmpty(ClusteringComparator comparator) + { + return isEmpty(comparator, start(), end()); + } + + /** + * Return whether the slice formed by the two provided bound is empty or not. + * + * @param comparator the comparator to compare the bounds. + * @param start the start for the slice to consider. This must be a start bound. + * @param end the end for the slice to consider. This must be an end bound. + * @return whether the slice formed by {@code start} and {@code end} is + * empty or not. + */ + public static boolean isEmpty(ClusteringComparator comparator, Slice.Bound start, Slice.Bound end) + { + assert start.isStart() && end.isEnd(); + return comparator.compare(end, start) < 0; + } + + /** + * Returns whether a given clustering is selected by this slice. + * + * @param comparator the comparator for the table this is a slice of. + * @param clustering the clustering to test inclusion of. + * + * @return whether {@code clustering} is selected by this slice. + */ + public boolean selects(ClusteringComparator comparator, Clustering clustering) + { + return comparator.compare(start, clustering) <= 0 && comparator.compare(clustering, end) <= 0; + } + + /** + * Returns whether a given bound is included in this slice. + * + * @param comparator the comparator for the table this is a slice of. + * @param bound the bound to test inclusion of. + * + * @return whether {@code bound} is within the bounds of this slice. + */ + public boolean includes(ClusteringComparator comparator, Bound bound) + { + return comparator.compare(start, bound) <= 0 && comparator.compare(bound, end) <= 0; + } + + /** + * Returns a slice for continuing paging from the last returned clustering prefix. + * + * @param comparator the comparator for the table this is a filter for. + * @param lastReturned the last clustering that was returned for the query we are paging for. The + * resulting slices will be such that only results coming stricly after {@code lastReturned} are returned + * (where coming after means "greater than" if {@code !reversed} and "lesser than" otherwise). + * @param inclusive whether or not we want to include the {@code lastReturned} in the newly returned page of results. + * @param reversed whether the query we're paging for is reversed or not. + * + * @return a new slice that selects results coming after {@code lastReturned}, or {@code null} if paging + * the resulting slice selects nothing (i.e. if it originally selects nothing coming after {@code lastReturned}). + */ + public Slice forPaging(ClusteringComparator comparator, Clustering lastReturned, boolean inclusive, boolean reversed) + { + if (reversed) + { + int cmp = comparator.compare(lastReturned, start); + if (cmp < 0 || (!inclusive && cmp == 0)) + return null; + + cmp = comparator.compare(end, lastReturned); + if (cmp < 0 || (inclusive && cmp == 0)) + return this; + + ByteBuffer[] values = extractValues(lastReturned); + return new Slice(start, inclusive ? Bound.inclusiveEndOf(values) : Bound.exclusiveEndOf(values)); + } + else + { + int cmp = comparator.compare(end, lastReturned); + if (cmp < 0 || (!inclusive && cmp == 0)) + return null; + + cmp = comparator.compare(lastReturned, start); + if (cmp < 0 || (inclusive && cmp == 0)) + return this; + + ByteBuffer[] values = extractValues(lastReturned); + return new Slice(inclusive ? Bound.inclusiveStartOf(values) : Bound.exclusiveStartOf(values), end); + } + } + + /** + * Given the per-clustering column minimum and maximum value a sstable contains, whether or not this slice potentially + * intersects that sstable or not. + * + * @param comparator the comparator for the table this is a slice of. + * @param minClusteringValues the smallest values for each clustering column that a sstable contains. + * @param maxClusteringValues the biggest values for each clustering column that a sstable contains. + * + * @return whether the slice might intersects with the sstable having {@code minClusteringValues} and + * {@code maxClusteringValues}. + */ + public boolean intersects(ClusteringComparator comparator, List minClusteringValues, List maxClusteringValues) + { + // If this slice start after max or end before min, it can't intersect + if (start.compareTo(comparator, maxClusteringValues) > 0 || end.compareTo(comparator, minClusteringValues) < 0) + return false; + + // We could safely return true here, but there's a minor optimization: if the first component + // of the slice is restricted to a single value (typically the slice is [4:5, 4:7]), we can + // check that the second component falls within the min/max for that component (and repeat for + // all components). + for (int j = 0; j < minClusteringValues.size() && j < maxClusteringValues.size(); j++) + { + ByteBuffer s = j < start.size() ? start.get(j) : null; + ByteBuffer f = j < end.size() ? end.get(j) : null; + + // we already know the first component falls within its min/max range (otherwise we wouldn't get here) + if (j > 0 && (j < end.size() && comparator.compareComponent(j, f, minClusteringValues.get(j)) < 0 || + j < start.size() && comparator.compareComponent(j, s, maxClusteringValues.get(j)) > 0)) + return false; + + // if this component isn't equal in the start and finish, we don't need to check any more + if (j >= start.size() || j >= end.size() || comparator.compareComponent(j, s, f) != 0) + break; + } + return true; + } + + public String toString(CFMetaData metadata) + { + return toString(metadata.comparator); + } + + public String toString(ClusteringComparator comparator) + { + StringBuilder sb = new StringBuilder(); + sb.append(start.isInclusive() ? "[" : "("); + for (int i = 0; i < start.size(); i++) + { + if (i > 0) + sb.append(":"); + sb.append(comparator.subtype(i).getString(start.get(i))); + } + sb.append(", "); + for (int i = 0; i < end.size(); i++) + { + if (i > 0) + sb.append(":"); + sb.append(comparator.subtype(i).getString(end.get(i))); + } + sb.append(end.isInclusive() ? "]" : ")"); + return sb.toString(); + } + + @Override + public boolean equals(Object other) + { + if(!(other instanceof Slice)) + return false; + + Slice that = (Slice)other; + return this.start().equals(that.start()) + && this.end().equals(that.end()); + } + + @Override + public int hashCode() + { + return Objects.hash(start(), end()); + } + + public static class Serializer + { + public void serialize(Slice slice, DataOutputPlus out, int version, List> types) throws IOException + { + Bound.serializer.serialize(slice.start, out, version, types); + Bound.serializer.serialize(slice.end, out, version, types); + } + + public long serializedSize(Slice slice, int version, List> types, TypeSizes sizes) + { + return Bound.serializer.serializedSize(slice.start, version, types, sizes) + + Bound.serializer.serializedSize(slice.end, version, types, sizes); + } + + public Slice deserialize(DataInput in, int version, List> types) throws IOException + { + Bound start = Bound.serializer.deserialize(in, version, types); + Bound end = Bound.serializer.deserialize(in, version, types); + return new Slice(start, end); + } + } + + /** + * The bound of a slice. + *

+ * This can be either a start or an end bound, and this can be either inclusive or exclusive. + */ + public static class Bound extends AbstractClusteringPrefix + { + private static final long EMPTY_SIZE = ObjectSizes.measure(new Bound(Kind.INCL_START_BOUND, new ByteBuffer[0])); + public static final Serializer serializer = new Serializer(); + + /** The smallest start bound, i.e. the one that starts before any row. */ + public static final Bound BOTTOM = inclusiveStartOf(); + /** The biggest end bound, i.e. the one that ends after any row. */ + public static final Bound TOP = inclusiveEndOf(); + + protected final Kind kind; + protected final ByteBuffer[] values; + + protected Bound(Kind kind, ByteBuffer[] values) + { + this.kind = kind; + this.values = values; + } + + public static Bound create(Kind kind, ByteBuffer[] values) + { + assert !kind.isBoundary(); + return new Bound(kind, values); + } + + public static Kind boundKind(boolean isStart, boolean isInclusive) + { + return isStart + ? (isInclusive ? Kind.INCL_START_BOUND : Kind.EXCL_START_BOUND) + : (isInclusive ? Kind.INCL_END_BOUND : Kind.EXCL_END_BOUND); + } + + public static Bound inclusiveStartOf(ByteBuffer... values) + { + return create(Kind.INCL_START_BOUND, values); + } + + public static Bound inclusiveEndOf(ByteBuffer... values) + { + return create(Kind.INCL_END_BOUND, values); + } + + public static Bound exclusiveStartOf(ByteBuffer... values) + { + return create(Kind.EXCL_START_BOUND, values); + } + + public static Bound exclusiveEndOf(ByteBuffer... values) + { + return create(Kind.EXCL_END_BOUND, values); + } + + public static Bound exclusiveStartOf(ClusteringPrefix prefix) + { + ByteBuffer[] values = new ByteBuffer[prefix.size()]; + for (int i = 0; i < prefix.size(); i++) + values[i] = prefix.get(i); + return exclusiveStartOf(values); + } + + public static Bound inclusiveEndOf(ClusteringPrefix prefix) + { + ByteBuffer[] values = new ByteBuffer[prefix.size()]; + for (int i = 0; i < prefix.size(); i++) + values[i] = prefix.get(i); + return inclusiveEndOf(values); + } + + public static Bound create(ClusteringComparator comparator, boolean isStart, boolean isInclusive, Object... values) + { + CBuilder builder = CBuilder.create(comparator); + for (int i = 0; i < values.length; i++) + { + Object val = values[i]; + if (val instanceof ByteBuffer) + builder.add((ByteBuffer)val); + else + builder.add(val); + } + return builder.buildBound(isStart, isInclusive); + } + + public Kind kind() + { + return kind; + } + + public int size() + { + return values.length; + } + + public ByteBuffer get(int i) + { + return values[i]; + } + + public Bound withNewKind(Kind kind) + { + assert !kind.isBoundary(); + return new Bound(kind, values); + } + + public boolean isStart() + { + return kind().isStart(); + } + + public boolean isEnd() + { + return !isStart(); + } + + public boolean isInclusive() + { + return kind == Kind.INCL_START_BOUND || kind == Kind.INCL_END_BOUND; + } + + public boolean isExclusive() + { + return kind == Kind.EXCL_START_BOUND || kind == Kind.EXCL_END_BOUND; + } + + /** + * Returns the inverse of the current bound. + *

+ * This invert both start into end (and vice-versa) and inclusive into exclusive (and vice-versa). + * + * @return the invert of this bound. For instance, if this bound is an exlusive start, this return + * an inclusive end with the same values. + */ + public Slice.Bound invert() + { + return withNewKind(kind().invert()); + } + + public ByteBuffer[] getRawValues() + { + return values; + } + + public void digest(MessageDigest digest) + { + for (int i = 0; i < size(); i++) + digest.update(get(i).duplicate()); + FBUtilities.updateWithByte(digest, kind().ordinal()); + } + + public void writeTo(Slice.Bound.Writer writer) + { + super.writeTo(writer); + writer.writeBoundKind(kind()); + } + + // For use by intersects, it's called with the sstable bound opposite to the slice bound + // (so if the slice bound is a start, it's call with the max sstable bound) + private int compareTo(ClusteringComparator comparator, List sstableBound) + { + for (int i = 0; i < sstableBound.size(); i++) + { + // Say the slice bound is a start. It means we're in the case where the max + // sstable bound is say (1:5) while the slice start is (1). So the start + // does start before the sstable end bound (and intersect it). It's the exact + // inverse with a end slice bound. + if (i >= size()) + return isStart() ? -1 : 1; + + int cmp = comparator.compareComponent(i, get(i), sstableBound.get(i)); + if (cmp != 0) + return cmp; + } + + // Say the slice bound is a start. I means we're in the case where the max + // sstable bound is say (1), while the slice start is (1:5). This again means + // that the slice start before the end bound. + if (size() > sstableBound.size()) + return isStart() ? -1 : 1; + + // The slice bound is equal to the sstable bound. Results depends on whether the slice is inclusive or not + return isInclusive() ? 0 : (isStart() ? 1 : -1); + } + + public String toString(CFMetaData metadata) + { + return toString(metadata.comparator); + } + + public String toString(ClusteringComparator comparator) + { + StringBuilder sb = new StringBuilder(); + sb.append(kind()).append("("); + for (int i = 0; i < size(); i++) + { + if (i > 0) + sb.append(", "); + sb.append(comparator.subtype(i).getString(get(i))); + } + return sb.append(")").toString(); + } + + // Overriding to get a more precise type + @Override + public Bound takeAlias() + { + return this; + } + + @Override + public long unsharedHeapSize() + { + return EMPTY_SIZE + ObjectSizes.sizeOnHeapOf(values); + } + + public long unsharedHeapSizeExcludingData() + { + return EMPTY_SIZE + ObjectSizes.sizeOnHeapExcludingData(values); + } + + public static Builder builder(int size) + { + return new Builder(size); + } + + public interface Writer extends ClusteringPrefix.Writer + { + public void writeBoundKind(Kind kind); + } + + public static class Builder implements Writer + { + private final ByteBuffer[] values; + private Kind kind; + private int idx; + + private Builder(int size) + { + this.values = new ByteBuffer[size]; + } + + public void writeClusteringValue(ByteBuffer value) + { + values[idx++] = value; + } + + public void writeBoundKind(Kind kind) + { + this.kind = kind; + } + + public Slice.Bound build() + { + assert idx == values.length; + return Slice.Bound.create(kind, values); + } + } + + /** + * Serializer for slice bounds. + *

+ * Contrarily to {@code Clustering}, a slice bound can only be a true prefix of the full clustering, so we actually record + * its size. + */ + public static class Serializer + { + public void serialize(Slice.Bound bound, DataOutputPlus out, int version, List> types) throws IOException + { + out.writeByte(bound.kind().ordinal()); + out.writeShort(bound.size()); + ClusteringPrefix.serializer.serializeValuesWithoutSize(bound, out, version, types); + } + + public long serializedSize(Slice.Bound bound, int version, List> types, TypeSizes sizes) + { + return 1 // kind ordinal + + sizes.sizeof((short)bound.size()) + + ClusteringPrefix.serializer.valuesWithoutSizeSerializedSize(bound, version, types, sizes); + } + + public Slice.Bound deserialize(DataInput in, int version, List> types) throws IOException + { + Kind kind = Kind.values()[in.readByte()]; + return deserializeValues(in, kind, version, types); + } + + public Slice.Bound deserializeValues(DataInput in, Kind kind, int version, List> types) throws IOException + { + int size = in.readUnsignedShort(); + if (size == 0) + return kind.isStart() ? BOTTOM : TOP; + + Builder builder = builder(size); + ClusteringPrefix.serializer.deserializeValuesWithoutSize(in, size, version, types, builder); + builder.writeBoundKind(kind); + return builder.build(); + } + + public void deserializeValues(DataInput in, Bound.Kind kind, int version, List> types, Writer writer) throws IOException + { + int size = in.readUnsignedShort(); + ClusteringPrefix.serializer.deserializeValuesWithoutSize(in, size, version, types, writer); + writer.writeBoundKind(kind); + } + + } + } +} diff --git a/src/java/org/apache/cassandra/db/SliceByNamesReadCommand.java b/src/java/org/apache/cassandra/db/SliceByNamesReadCommand.java deleted file mode 100644 index 65eefaa97d..0000000000 --- a/src/java/org/apache/cassandra/db/SliceByNamesReadCommand.java +++ /dev/null @@ -1,125 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.apache.cassandra.db; - -import java.io.*; -import java.nio.ByteBuffer; - -import com.google.common.base.Objects; - -import org.apache.cassandra.config.CFMetaData; -import org.apache.cassandra.config.Schema; -import org.apache.cassandra.db.filter.*; -import org.apache.cassandra.io.IVersionedSerializer; -import org.apache.cassandra.io.util.DataOutputPlus; -import org.apache.cassandra.service.StorageService; -import org.apache.cassandra.utils.ByteBufferUtil; - -public class SliceByNamesReadCommand extends ReadCommand -{ - static final SliceByNamesReadCommandSerializer serializer = new SliceByNamesReadCommandSerializer(); - - public final NamesQueryFilter filter; - - public SliceByNamesReadCommand(String keyspaceName, ByteBuffer key, String cfName, long timestamp, NamesQueryFilter filter) - { - super(keyspaceName, key, cfName, timestamp, Type.GET_BY_NAMES); - this.filter = filter; - } - - public ReadCommand copy() - { - return new SliceByNamesReadCommand(ksName, key, cfName, timestamp, filter).setIsDigestQuery(isDigestQuery()); - } - - public Row getRow(Keyspace keyspace) - { - DecoratedKey dk = StorageService.getPartitioner().decorateKey(key); - return keyspace.getRow(new QueryFilter(dk, cfName, filter, timestamp)); - } - - @Override - public String toString() - { - return Objects.toStringHelper(this) - .add("ksName", ksName) - .add("cfName", cfName) - .add("key", ByteBufferUtil.bytesToHex(key)) - .add("filter", filter) - .add("timestamp", timestamp) - .toString(); - } - - public IDiskAtomFilter filter() - { - return filter; - } -} - -class SliceByNamesReadCommandSerializer implements IVersionedSerializer -{ - public void serialize(ReadCommand cmd, DataOutputPlus out, int version) throws IOException - { - SliceByNamesReadCommand command = (SliceByNamesReadCommand) cmd; - out.writeBoolean(command.isDigestQuery()); - out.writeUTF(command.ksName); - ByteBufferUtil.writeWithShortLength(command.key, out); - out.writeUTF(command.cfName); - out.writeLong(cmd.timestamp); - - CFMetaData metadata = Schema.instance.getCFMetaData(cmd.ksName, cmd.cfName); - metadata.comparator.namesQueryFilterSerializer().serialize(command.filter, out, version); - } - - public ReadCommand deserialize(DataInput in, int version) throws IOException - { - boolean isDigest = in.readBoolean(); - String keyspaceName = in.readUTF(); - ByteBuffer key = ByteBufferUtil.readWithShortLength(in); - String cfName = in.readUTF(); - long timestamp = in.readLong(); - CFMetaData metadata = Schema.instance.getCFMetaData(keyspaceName, cfName); - if (metadata == null) - { - String message = String.format("Got slice command for nonexistent table %s.%s. If the table was just " + - "created, this is likely due to the schema not being fully propagated. Please wait for schema " + - "agreement on table creation.", keyspaceName, cfName); - throw new UnknownColumnFamilyException(message, null); - } - NamesQueryFilter filter = metadata.comparator.namesQueryFilterSerializer().deserialize(in, version); - return new SliceByNamesReadCommand(keyspaceName, key, cfName, timestamp, filter).setIsDigestQuery(isDigest); - } - - public long serializedSize(ReadCommand cmd, int version) - { - TypeSizes sizes = TypeSizes.NATIVE; - SliceByNamesReadCommand command = (SliceByNamesReadCommand) cmd; - int size = sizes.sizeof(command.isDigestQuery()); - int keySize = command.key.remaining(); - - CFMetaData metadata = Schema.instance.getCFMetaData(cmd.ksName, cmd.cfName); - - size += sizes.sizeof(command.ksName); - size += sizes.sizeof((short)keySize) + keySize; - size += sizes.sizeof(command.cfName); - size += sizes.sizeof(cmd.timestamp); - size += metadata.comparator.namesQueryFilterSerializer().serializedSize(command.filter, version); - - return size; - } -} diff --git a/src/java/org/apache/cassandra/db/SliceFromReadCommand.java b/src/java/org/apache/cassandra/db/SliceFromReadCommand.java deleted file mode 100644 index 699519380b..0000000000 --- a/src/java/org/apache/cassandra/db/SliceFromReadCommand.java +++ /dev/null @@ -1,207 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.apache.cassandra.db; - -import java.io.DataInput; -import java.io.IOException; -import java.nio.ByteBuffer; - -import com.google.common.base.Objects; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; - -import org.apache.cassandra.config.CFMetaData; -import org.apache.cassandra.config.Schema; -import org.apache.cassandra.db.filter.IDiskAtomFilter; -import org.apache.cassandra.db.filter.QueryFilter; -import org.apache.cassandra.db.filter.SliceQueryFilter; -import org.apache.cassandra.io.IVersionedSerializer; -import org.apache.cassandra.io.util.DataOutputPlus; -import org.apache.cassandra.service.RowDataResolver; -import org.apache.cassandra.service.StorageService; -import org.apache.cassandra.utils.ByteBufferUtil; -import org.apache.cassandra.utils.Pair; - -public class SliceFromReadCommand extends ReadCommand -{ - private static final Logger logger = LoggerFactory.getLogger(SliceFromReadCommand.class); - - static final SliceFromReadCommandSerializer serializer = new SliceFromReadCommandSerializer(); - - public final SliceQueryFilter filter; - - public SliceFromReadCommand(String keyspaceName, ByteBuffer key, String cfName, long timestamp, SliceQueryFilter filter) - { - super(keyspaceName, key, cfName, timestamp, Type.GET_SLICES); - this.filter = filter; - } - - public ReadCommand copy() - { - return new SliceFromReadCommand(ksName, key, cfName, timestamp, filter).setIsDigestQuery(isDigestQuery()); - } - - public Row getRow(Keyspace keyspace) - { - CFMetaData cfm = Schema.instance.getCFMetaData(ksName, cfName); - DecoratedKey dk = StorageService.getPartitioner().decorateKey(key); - - // If we're doing a reversed query and the filter includes static columns, we need to issue two separate - // reads in order to guarantee that the static columns are fetched. See CASSANDRA-8502 for more details. - if (filter.reversed && filter.hasStaticSlice(cfm)) - { - logger.debug("Splitting reversed slice with static columns into two reads"); - Pair newFilters = filter.splitOutStaticSlice(cfm); - - Row normalResults = keyspace.getRow(new QueryFilter(dk, cfName, newFilters.right, timestamp)); - Row staticResults = keyspace.getRow(new QueryFilter(dk, cfName, newFilters.left, timestamp)); - - // add the static results to the start of the normal results - if (normalResults.cf == null) - return staticResults; - - if (staticResults.cf != null) - for (Cell cell : staticResults.cf.getReverseSortedColumns()) - normalResults.cf.addColumn(cell); - - return normalResults; - } - - return keyspace.getRow(new QueryFilter(dk, cfName, filter, timestamp)); - } - - @Override - public ReadCommand maybeGenerateRetryCommand(RowDataResolver resolver, Row row) - { - int maxLiveColumns = resolver.getMaxLiveCount(); - - int count = filter.count; - // We generate a retry if at least one node reply with count live columns but after merge we have less - // than the total number of column we are interested in (which may be < count on a retry). - // So in particular, if no host returned count live columns, we know it's not a short read. - if (maxLiveColumns < count) - return null; - - int liveCountInRow = row == null || row.cf == null ? 0 : filter.getLiveCount(row.cf, timestamp); - if (liveCountInRow < getOriginalRequestedCount()) - { - // We asked t (= count) live columns and got l (=liveCountInRow) ones. - // From that, we can estimate that on this row, for x requested - // columns, only l/t end up live after reconciliation. So for next - // round we want to ask x column so that x * (l/t) == t, i.e. x = t^2/l. - int retryCount = liveCountInRow == 0 ? count + 1 : ((count * count) / liveCountInRow) + 1; - SliceQueryFilter newFilter = filter.withUpdatedCount(retryCount); - return new RetriedSliceFromReadCommand(ksName, key, cfName, timestamp, newFilter, getOriginalRequestedCount()); - } - - return null; - } - - @Override - public void maybeTrim(Row row) - { - if ((row == null) || (row.cf == null)) - return; - - filter.trim(row.cf, getOriginalRequestedCount(), timestamp); - } - - public IDiskAtomFilter filter() - { - return filter; - } - - public SliceFromReadCommand withUpdatedFilter(SliceQueryFilter newFilter) - { - return new SliceFromReadCommand(ksName, key, cfName, timestamp, newFilter); - } - - /** - * The original number of columns requested by the user. - * This can be different from count when the slice command is a retry (see - * RetriedSliceFromReadCommand) - */ - protected int getOriginalRequestedCount() - { - return filter.count; - } - - @Override - public String toString() - { - return Objects.toStringHelper(this) - .add("ksName", ksName) - .add("cfName", cfName) - .add("key", ByteBufferUtil.bytesToHex(key)) - .add("filter", filter) - .add("timestamp", timestamp) - .toString(); - } -} - -class SliceFromReadCommandSerializer implements IVersionedSerializer -{ - public void serialize(ReadCommand rm, DataOutputPlus out, int version) throws IOException - { - SliceFromReadCommand realRM = (SliceFromReadCommand)rm; - out.writeBoolean(realRM.isDigestQuery()); - out.writeUTF(realRM.ksName); - ByteBufferUtil.writeWithShortLength(realRM.key, out); - out.writeUTF(realRM.cfName); - out.writeLong(realRM.timestamp); - CFMetaData metadata = Schema.instance.getCFMetaData(realRM.ksName, realRM.cfName); - metadata.comparator.sliceQueryFilterSerializer().serialize(realRM.filter, out, version); - } - - public ReadCommand deserialize(DataInput in, int version) throws IOException - { - boolean isDigest = in.readBoolean(); - String keyspaceName = in.readUTF(); - ByteBuffer key = ByteBufferUtil.readWithShortLength(in); - String cfName = in.readUTF(); - long timestamp = in.readLong(); - CFMetaData metadata = Schema.instance.getCFMetaData(keyspaceName, cfName); - if (metadata == null) - { - String message = String.format("Got slice command for nonexistent table %s.%s. If the table was just " + - "created, this is likely due to the schema not being fully propagated. Please wait for schema " + - "agreement on table creation.", keyspaceName, cfName); - throw new UnknownColumnFamilyException(message, null); - } - SliceQueryFilter filter = metadata.comparator.sliceQueryFilterSerializer().deserialize(in, version); - return new SliceFromReadCommand(keyspaceName, key, cfName, timestamp, filter).setIsDigestQuery(isDigest); - } - - public long serializedSize(ReadCommand cmd, int version) - { - TypeSizes sizes = TypeSizes.NATIVE; - SliceFromReadCommand command = (SliceFromReadCommand) cmd; - int keySize = command.key.remaining(); - - CFMetaData metadata = Schema.instance.getCFMetaData(cmd.ksName, cmd.cfName); - - int size = sizes.sizeof(cmd.isDigestQuery()); // boolean - size += sizes.sizeof(command.ksName); - size += sizes.sizeof((short) keySize) + keySize; - size += sizes.sizeof(command.cfName); - size += sizes.sizeof(cmd.timestamp); - size += metadata.comparator.sliceQueryFilterSerializer().serializedSize(command.filter, version); - - return size; - } -} diff --git a/src/java/org/apache/cassandra/db/Slices.java b/src/java/org/apache/cassandra/db/Slices.java new file mode 100644 index 0000000000..ec7797d6d0 --- /dev/null +++ b/src/java/org/apache/cassandra/db/Slices.java @@ -0,0 +1,898 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.cassandra.db; + +import java.io.DataInput; +import java.io.IOException; +import java.nio.ByteBuffer; +import java.util.*; + +import com.google.common.collect.Iterators; + +import org.apache.cassandra.config.CFMetaData; +import org.apache.cassandra.config.ColumnDefinition; +import org.apache.cassandra.db.rows.*; +import org.apache.cassandra.db.marshal.AbstractType; +import org.apache.cassandra.io.util.DataOutputPlus; + +/** + * Represents the selection of multiple range of rows within a partition. + *

+ * A {@code Slices} is basically a list of {@code Slice}, though those are guaranteed to be non-overlapping + * and always in clustering order. + */ +public abstract class Slices implements Iterable +{ + public static final Serializer serializer = new Serializer(); + + /** Slices selecting all the rows of a partition. */ + public static final Slices ALL = new SelectAllSlices(); + /** Slices selecting no rows in a partition. */ + public static final Slices NONE = new SelectNoSlices(); + + protected Slices() + { + } + + /** + * Creates a {@code Slices} object that contains a single slice. + * + * @param comparator the comparator for the table {@code slice} is a slice of. + * @param slice the single slice that the return object should contains. + * + * @return the newly created {@code Slices} object. + */ + public static Slices with(ClusteringComparator comparator, Slice slice) + { + if (slice.start() == Slice.Bound.BOTTOM && slice.end() == Slice.Bound.TOP) + return Slices.ALL; + + assert comparator.compare(slice.start(), slice.end()) <= 0; + return new ArrayBackedSlices(comparator, new Slice[]{ slice }); + } + + /** + * Whether the slices has a lower bound, that is whether it's first slice start is {@code Slice.BOTTOM}. + * + * @return whether the slices has a lower bound. + */ + public abstract boolean hasLowerBound(); + + /** + * Whether the slices has an upper bound, that is whether it's last slice end is {@code Slice.TOP}. + * + * @return whether the slices has an upper bound. + */ + public abstract boolean hasUpperBound(); + + /** + * The number of slice this object contains. + * + * @return the number of slice this object contains. + */ + public abstract int size(); + + /** + * Returns the ith slice of this {@code Slices} object. + * + * @return the ith slice of this object. + */ + public abstract Slice get(int i); + + /** + * Returns slices for continuing the paging of those slices given the last returned clustering prefix. + * + * @param comparator the comparator for the table this is a filter for. + * @param lastReturned the last clustering that was returned for the query we are paging for. The + * resulting slices will be such that only results coming stricly after {@code lastReturned} are returned + * (where coming after means "greater than" if {@code !reversed} and "lesser than" otherwise). + * @param inclusive whether or not we want to include the {@code lastReturned} in the newly returned page of results. + * @param reversed whether the query we're paging for is reversed or not. + * + * @return new slices that select results coming after {@code lastReturned}. + */ + public abstract Slices forPaging(ClusteringComparator comparator, Clustering lastReturned, boolean inclusive, boolean reversed); + + /** + * An object that allows to test whether rows are selected by this {@code Slices} objects assuming those rows + * are tested in clustering order. + * + * @param reversed if true, the rows passed to the returned object will be assumed to be in reversed clustering + * order, otherwise they should be in clustering order. + * + * @return an object that tests for selection of rows by this {@code Slices} object. + */ + public abstract InOrderTester inOrderTester(boolean reversed); + + /** + * Whether a given clustering (row) is selected by this {@code Slices} object. + * + * @param clustering the clustering to test for selection. + * + * @return whether a given clustering (row) is selected by this {@code Slices} object. + */ + public abstract boolean selects(Clustering clustering); + + + /** + * Given the per-clustering column minimum and maximum value a sstable contains, whether or not this slices potentially + * intersects that sstable or not. + * + * @param minClusteringValues the smallest values for each clustering column that a sstable contains. + * @param maxClusteringValues the biggest values for each clustering column that a sstable contains. + * + * @return whether the slices might intersects with the sstable having {@code minClusteringValues} and + * {@code maxClusteringValues}. + */ + public abstract boolean intersects(List minClusteringValues, List maxClusteringValues); + + /** + * Given a sliceable row iterator, returns a row iterator that only return rows selected by the slice of + * this {@code Slices} object. + * + * @param iter the sliceable iterator to filter. + * + * @return an iterator that only returns the rows (or rather Unfiltered) of {@code iter} that are selected by those slices. + */ + public abstract UnfilteredRowIterator makeSliceIterator(SliceableUnfilteredRowIterator iter); + + public abstract String toCQLString(CFMetaData metadata); + + /** + * In simple object that allows to test the inclusion of rows in those slices assuming those rows + * are passed (to {@link #includes}) in clustering order (or reverse clustering ordered, depending + * of the argument passed to {@link #inOrderTester}). + */ + public interface InOrderTester + { + public boolean includes(Clustering value); + public boolean isDone(); + } + + /** + * Builder to create {@code Slices} objects. + */ + public static class Builder + { + private final ClusteringComparator comparator; + + private final List slices; + + private boolean needsNormalizing; + + public Builder(ClusteringComparator comparator) + { + this.comparator = comparator; + this.slices = new ArrayList<>(); + } + + public Builder(ClusteringComparator comparator, int initialSize) + { + this.comparator = comparator; + this.slices = new ArrayList<>(initialSize); + } + + public Builder add(Slice.Bound start, Slice.Bound end) + { + return add(Slice.make(start, end)); + } + + public Builder add(Slice slice) + { + assert comparator.compare(slice.start(), slice.end()) <= 0; + if (slices.size() > 0 && comparator.compare(slices.get(slices.size()-1).end(), slice.start()) > 0) + needsNormalizing = true; + slices.add(slice); + return this; + } + + public int size() + { + return slices.size(); + } + + public Slices build() + { + if (slices.isEmpty()) + return NONE; + + if (slices.size() == 1 && slices.get(0) == Slice.ALL) + return ALL; + + List normalized = needsNormalizing + ? normalize(slices) + : slices; + + return new ArrayBackedSlices(comparator, normalized.toArray(new Slice[normalized.size()])); + } + + /** + * Given an array of slices (potentially overlapping and in any order) and return an equivalent array + * of non-overlapping slices in clustering order. + * + * @param slices an array of slices. This may be modified by this method. + * @return the smallest possible array of non-overlapping slices in clustering order. If the original + * slices are already non-overlapping and in comparator order, this may or may not return the provided slices + * directly. + */ + private List normalize(List slices) + { + if (slices.size() <= 1) + return slices; + + Collections.sort(slices, new Comparator() + { + @Override + public int compare(Slice s1, Slice s2) + { + int c = comparator.compare(s1.start(), s2.start()); + if (c != 0) + return c; + + return comparator.compare(s1.end(), s2.end()); + } + }); + + List slicesCopy = new ArrayList<>(slices.size()); + + Slice last = slices.get(0); + + for (int i = 1; i < slices.size(); i++) + { + Slice s2 = slices.get(i); + + boolean includesStart = last.includes(comparator, s2.start()); + boolean includesFinish = last.includes(comparator, s2.end()); + + if (includesStart && includesFinish) + continue; + + if (!includesStart && !includesFinish) + { + slicesCopy.add(last); + last = s2; + continue; + } + + if (includesStart) + { + last = Slice.make(last.start(), s2.end()); + continue; + } + + assert !includesFinish; + } + + slicesCopy.add(last); + return slicesCopy; + } + } + + public static class Serializer + { + public void serialize(Slices slices, DataOutputPlus out, int version) throws IOException + { + int size = slices.size(); + out.writeInt(size); + + if (size == 0) + return; + + List> types = slices == ALL + ? Collections.>emptyList() + : ((ArrayBackedSlices)slices).comparator.subtypes(); + + for (Slice slice : slices) + Slice.serializer.serialize(slice, out, version, types); + } + + public long serializedSize(Slices slices, int version, TypeSizes sizes) + { + long size = sizes.sizeof(slices.size()); + + if (slices.size() == 0) + return size; + + List> types = slices instanceof SelectAllSlices + ? Collections.>emptyList() + : ((ArrayBackedSlices)slices).comparator.subtypes(); + + for (Slice slice : slices) + size += Slice.serializer.serializedSize(slice, version, types, sizes); + + return size; + } + + public Slices deserialize(DataInput in, int version, CFMetaData metadata) throws IOException + { + int size = in.readInt(); + + if (size == 0) + return NONE; + + Slice[] slices = new Slice[size]; + for (int i = 0; i < size; i++) + slices[i] = Slice.serializer.deserialize(in, version, metadata.comparator.subtypes()); + + if (size == 1 && slices[0].start() == Slice.Bound.BOTTOM && slices[0].end() == Slice.Bound.TOP) + return ALL; + + return new ArrayBackedSlices(metadata.comparator, slices); + } + } + + /** + * Simple {@code Slices} implementation that stores its slices in an array. + */ + private static class ArrayBackedSlices extends Slices + { + private final ClusteringComparator comparator; + + private final Slice[] slices; + + private ArrayBackedSlices(ClusteringComparator comparator, Slice[] slices) + { + this.comparator = comparator; + this.slices = slices; + } + + public int size() + { + return slices.length; + } + + public boolean hasLowerBound() + { + return slices[0].start().size() != 0; + } + + public boolean hasUpperBound() + { + return slices[slices.length - 1].end().size() != 0; + } + + public Slice get(int i) + { + return slices[i]; + } + + public boolean selects(Clustering clustering) + { + for (int i = 0; i < slices.length; i++) + { + Slice slice = slices[i]; + if (comparator.compare(clustering, slice.start()) < 0) + return false; + + if (comparator.compare(clustering, slice.end()) <= 0) + return true; + } + return false; + } + + public InOrderTester inOrderTester(boolean reversed) + { + return reversed ? new InReverseOrderTester() : new InForwardOrderTester(); + } + + public Slices forPaging(ClusteringComparator comparator, Clustering lastReturned, boolean inclusive, boolean reversed) + { + return reversed ? forReversePaging(comparator, lastReturned, inclusive) : forForwardPaging(comparator, lastReturned, inclusive); + } + + private Slices forForwardPaging(ClusteringComparator comparator, Clustering lastReturned, boolean inclusive) + { + for (int i = 0; i < slices.length; i++) + { + Slice slice = slices[i]; + Slice newSlice = slice.forPaging(comparator, lastReturned, inclusive, false); + if (newSlice == null) + continue; + + if (slice == newSlice && i == 0) + return this; + + ArrayBackedSlices newSlices = new ArrayBackedSlices(comparator, Arrays.copyOfRange(slices, i, slices.length)); + newSlices.slices[0] = newSlice; + return newSlices; + } + return Slices.NONE; + } + + private Slices forReversePaging(ClusteringComparator comparator, Clustering lastReturned, boolean inclusive) + { + for (int i = slices.length - 1; i >= 0; i--) + { + Slice slice = slices[i]; + Slice newSlice = slice.forPaging(comparator, lastReturned, inclusive, true); + if (newSlice == null) + continue; + + if (slice == newSlice && i == slices.length - 1) + return this; + + ArrayBackedSlices newSlices = new ArrayBackedSlices(comparator, Arrays.copyOfRange(slices, 0, i + 1)); + newSlices.slices[i] = newSlice; + return newSlices; + } + return Slices.NONE; + } + + public boolean intersects(List minClusteringValues, List maxClusteringValues) + { + for (Slice slice : this) + { + if (slice.intersects(comparator, minClusteringValues, maxClusteringValues)) + return true; + } + return false; + } + + public UnfilteredRowIterator makeSliceIterator(final SliceableUnfilteredRowIterator iter) + { + return new WrappingUnfilteredRowIterator(iter) + { + private int nextSlice = iter.isReverseOrder() ? slices.length - 1 : 0; + private Iterator currentSliceIterator = Collections.emptyIterator(); + + private Unfiltered next; + + @Override + public boolean hasNext() + { + prepareNext(); + return next != null; + } + + @Override + public Unfiltered next() + { + prepareNext(); + Unfiltered toReturn = next; + next = null; + return toReturn; + } + + private boolean hasMoreSlice() + { + return isReverseOrder() + ? nextSlice >= 0 + : nextSlice < slices.length; + } + + private Slice popNextSlice() + { + return slices[isReverseOrder() ? nextSlice-- : nextSlice++]; + } + + private void prepareNext() + { + if (next != null) + return; + + while (true) + { + if (currentSliceIterator.hasNext()) + { + next = currentSliceIterator.next(); + return; + } + + if (!hasMoreSlice()) + return; + + currentSliceIterator = iter.slice(popNextSlice()); + } + } + }; + } + + public Iterator iterator() + { + return Iterators.forArray(slices); + } + + private class InForwardOrderTester implements InOrderTester + { + private int idx; + private boolean inSlice; + + public boolean includes(Clustering value) + { + while (idx < slices.length) + { + if (!inSlice) + { + int cmp = comparator.compare(value, slices[idx].start()); + // value < start + if (cmp < 0) + return false; + + inSlice = true; + + if (cmp == 0) + return true; + } + + // Here, start < value and inSlice + if (comparator.compare(value, slices[idx].end()) <= 0) + return true; + + ++idx; + inSlice = false; + } + return false; + } + + public boolean isDone() + { + return idx >= slices.length; + } + } + + private class InReverseOrderTester implements InOrderTester + { + private int idx; + private boolean inSlice; + + public InReverseOrderTester() + { + this.idx = slices.length - 1; + } + + public boolean includes(Clustering value) + { + while (idx >= 0) + { + if (!inSlice) + { + int cmp = comparator.compare(slices[idx].end(), value); + // value > end + if (cmp > 0) + return false; + + inSlice = true; + + if (cmp == 0) + return true; + } + + // Here, value <= end and inSlice + if (comparator.compare(slices[idx].start(), value) <= 0) + return true; + + --idx; + inSlice = false; + } + return false; + } + + public boolean isDone() + { + return idx < 0; + } + } + + @Override + public String toString() + { + StringBuilder sb = new StringBuilder(); + sb.append("{"); + for (int i = 0; i < slices.length; i++) + { + if (i > 0) + sb.append(", "); + sb.append(slices[i].toString(comparator)); + } + return sb.append("}").toString(); + } + + public String toCQLString(CFMetaData metadata) + { + StringBuilder sb = new StringBuilder(); + + // In CQL, condition are expressed by column, so first group things that way, + // i.e. for each column, we create a list of what each slice contains on that column + int clusteringSize = metadata.clusteringColumns().size(); + List> columnComponents = new ArrayList<>(clusteringSize); + for (int i = 0; i < clusteringSize; i++) + { + List perSlice = new ArrayList<>(); + columnComponents.add(perSlice); + + for (int j = 0; j < slices.length; j++) + { + ComponentOfSlice c = ComponentOfSlice.fromSlice(i, slices[j]); + if (c != null) + perSlice.add(c); + } + } + + boolean needAnd = false; + for (int i = 0; i < clusteringSize; i++) + { + ColumnDefinition column = metadata.clusteringColumns().get(i); + List componentInfo = columnComponents.get(i); + if (componentInfo.isEmpty()) + break; + + // For a given column, there is only 3 cases that CQL currently generates: + // 1) every slice are EQ with the same value, it's a simple '=' relation. + // 2) every slice are EQ but with different values, it's a IN relation. + // 3) every slice aren't EQ but have the same values, we have inequality relations. + // Note that this doesn't cover everything that ReadCommand can express, but + // as it's all that CQL support for now, we'll ignore other cases (which would then + // display a bogus query but that's not the end of the world). + // TODO: we should improve this at some point. + ComponentOfSlice first = componentInfo.get(0); + if (first.isEQ()) + { + if (needAnd) + sb.append(" AND "); + needAnd = true; + + sb.append(column.name); + + Set values = new LinkedHashSet<>(); + for (int j = 0; j < componentInfo.size(); j++) + values.add(componentInfo.get(j).startValue); + + if (values.size() == 1) + { + sb.append(" = ").append(column.type.getString(first.startValue)); + } + else + { + sb.append(" IN ("); + int j = 0; + for (ByteBuffer value : values) + sb.append(j++ == 0 ? "" : ", ").append(column.type.getString(value)); + sb.append(")"); + } + } + else + { + // As said above, we assume (without checking) that this means all ComponentOfSlice for this column + // are the same, so we only bother about the first. + if (first.startValue != null) + { + if (needAnd) + sb.append(" AND "); + needAnd = true; + sb.append(column.name).append(first.startInclusive ? " >= " : " > ").append(column.type.getString(first.startValue)); + } + if (first.endValue != null) + { + if (needAnd) + sb.append(" AND "); + needAnd = true; + sb.append(column.name).append(first.endInclusive ? " <= " : " < ").append(column.type.getString(first.endValue)); + } + } + } + return sb.toString(); + } + + // An somewhat adhoc utility class only used by toCQLString + private static class ComponentOfSlice + { + public final boolean startInclusive; + public final ByteBuffer startValue; + public final boolean endInclusive; + public final ByteBuffer endValue; + + private ComponentOfSlice(boolean startInclusive, ByteBuffer startValue, boolean endInclusive, ByteBuffer endValue) + { + this.startInclusive = startInclusive; + this.startValue = startValue; + this.endInclusive = endInclusive; + this.endValue = endValue; + } + + public static ComponentOfSlice fromSlice(int component, Slice slice) + { + Slice.Bound start = slice.start(); + Slice.Bound end = slice.end(); + + if (component >= start.size() && component >= end.size()) + return null; + + boolean startInclusive = true, endInclusive = true; + ByteBuffer startValue = null, endValue = null; + if (component < start.size()) + { + startInclusive = start.isInclusive(); + startValue = start.get(component); + } + if (component < end.size()) + { + endInclusive = end.isInclusive(); + endValue = end.get(component); + } + return new ComponentOfSlice(startInclusive, startValue, endInclusive, endValue); + } + + public boolean isEQ() + { + return startValue.equals(endValue); + } + } + } + + /** + * Specialized implementation of {@code Slices} that selects all rows. + *

+ * This is equivalent to having the single {@code Slice.ALL} slice, but is somewhat more effecient. + */ + private static class SelectAllSlices extends Slices + { + private static final InOrderTester trivialTester = new InOrderTester() + { + public boolean includes(Clustering value) + { + return true; + } + + public boolean isDone() + { + return false; + } + }; + + public int size() + { + return 1; + } + + public Slice get(int i) + { + return Slice.ALL; + } + + public boolean hasLowerBound() + { + return false; + } + + public boolean hasUpperBound() + { + return false; + } + + public boolean selects(Clustering clustering) + { + return true; + } + + public Slices forPaging(ClusteringComparator comparator, Clustering lastReturned, boolean inclusive, boolean reversed) + { + return new ArrayBackedSlices(comparator, new Slice[]{ Slice.ALL.forPaging(comparator, lastReturned, inclusive, reversed) }); + } + + public InOrderTester inOrderTester(boolean reversed) + { + return trivialTester; + } + + public boolean intersects(List minClusteringValues, List maxClusteringValues) + { + return true; + } + + public UnfilteredRowIterator makeSliceIterator(SliceableUnfilteredRowIterator iter) + { + return iter; + } + + public Iterator iterator() + { + return Iterators.singletonIterator(Slice.ALL); + } + + @Override + public String toString() + { + return "ALL"; + } + + public String toCQLString(CFMetaData metadata) + { + return ""; + } + } + + /** + * Specialized implementation of {@code Slices} that selects no rows. + */ + private static class SelectNoSlices extends Slices + { + private static final InOrderTester trivialTester = new InOrderTester() + { + public boolean includes(Clustering value) + { + return false; + } + + public boolean isDone() + { + return true; + } + }; + + public int size() + { + return 0; + } + + public Slice get(int i) + { + throw new UnsupportedOperationException(); + } + + public boolean hasLowerBound() + { + return false; + } + + public boolean hasUpperBound() + { + return false; + } + + public Slices forPaging(ClusteringComparator comparator, Clustering lastReturned, boolean inclusive, boolean reversed) + { + return this; + } + + public boolean selects(Clustering clustering) + { + return false; + } + + public InOrderTester inOrderTester(boolean reversed) + { + return trivialTester; + } + + public boolean intersects(List minClusteringValues, List maxClusteringValues) + { + return false; + } + + public UnfilteredRowIterator makeSliceIterator(SliceableUnfilteredRowIterator iter) + { + return UnfilteredRowIterators.emptyIterator(iter.metadata(), iter.partitionKey(), iter.isReverseOrder()); + } + + public Iterator iterator() + { + return Iterators.emptyIterator(); + } + + @Override + public String toString() + { + return "NONE"; + } + + public String toCQLString(CFMetaData metadata) + { + return ""; + } + } +} diff --git a/src/java/org/apache/cassandra/db/SuperColumns.java b/src/java/org/apache/cassandra/db/SuperColumns.java deleted file mode 100644 index 65e153f75e..0000000000 --- a/src/java/org/apache/cassandra/db/SuperColumns.java +++ /dev/null @@ -1,230 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.apache.cassandra.db; - -import java.io.DataInput; -import java.io.IOError; -import java.io.IOException; -import java.nio.ByteBuffer; -import java.util.ArrayList; -import java.util.Iterator; -import java.util.List; -import java.util.SortedSet; -import java.util.TreeSet; - -import org.apache.cassandra.config.CFMetaData; -import org.apache.cassandra.db.composites.*; -import org.apache.cassandra.db.filter.*; -import org.apache.cassandra.db.marshal.AbstractType; -import org.apache.cassandra.utils.ByteBufferUtil; - -public class SuperColumns -{ - public static Iterator onDiskIterator(DataInput in, int superColumnCount, ColumnSerializer.Flag flag, int expireBefore, CellNameType type) - { - return new SCIterator(in, superColumnCount, flag, expireBefore, type); - } - - public static void deserializerSuperColumnFamily(DataInput in, ColumnFamily cf, ColumnSerializer.Flag flag, int version) throws IOException - { - // Note that there was no way to insert a range tombstone in a SCF in 1.2 - cf.delete(cf.getComparator().deletionInfoSerializer().deserialize(in, version)); - assert !cf.deletionInfo().rangeIterator().hasNext(); - - Iterator iter = onDiskIterator(in, in.readInt(), flag, Integer.MIN_VALUE, cf.getComparator()); - while (iter.hasNext()) - cf.addAtom(iter.next()); - } - - private static class SCIterator implements Iterator - { - private final DataInput in; - private final int scCount; - - private final ColumnSerializer.Flag flag; - private final int expireBefore; - - private final CellNameType type; - - private int read; - private ByteBuffer scName; - private Iterator subColumnsIterator; - - private SCIterator(DataInput in, int superColumnCount, ColumnSerializer.Flag flag, int expireBefore, CellNameType type) - { - this.in = in; - this.scCount = superColumnCount; - this.flag = flag; - this.expireBefore = expireBefore; - this.type = type; - } - - public boolean hasNext() - { - return (subColumnsIterator != null && subColumnsIterator.hasNext()) || read < scCount; - } - - public OnDiskAtom next() - { - try - { - if (subColumnsIterator != null && subColumnsIterator.hasNext()) - { - Cell c = subColumnsIterator.next(); - return c.withUpdatedName(type.makeCellName(scName, c.name().toByteBuffer())); - } - - // Read one more super column - ++read; - - scName = ByteBufferUtil.readWithShortLength(in); - DeletionInfo delInfo = new DeletionInfo(DeletionTime.serializer.deserialize(in)); - - /* read the number of columns */ - int size = in.readInt(); - List subCells = new ArrayList<>(size); - - ColumnSerializer colSer = subType(type).columnSerializer(); - for (int i = 0; i < size; ++i) - subCells.add(colSer.deserialize(in, flag, expireBefore)); - - subColumnsIterator = subCells.iterator(); - - // If the SC was deleted, return that first, otherwise return the first subcolumn - DeletionTime dtime = delInfo.getTopLevelDeletion(); - if (!dtime.equals(DeletionTime.LIVE)) - return new RangeTombstone(startOf(scName), endOf(scName), dtime); - - return next(); - } - catch (IOException e) - { - throw new IOError(e); - } - } - - public void remove() - { - throw new UnsupportedOperationException(); - } - } - - private static CellNameType subType(CellNameType type) - { - return new SimpleDenseCellNameType(type.subtype(1)); - } - - public static CellNameType scNameType(CellNameType type) - { - return new SimpleDenseCellNameType(type.subtype(0)); - } - - public static AbstractType getComparatorFor(CFMetaData metadata, ByteBuffer superColumn) - { - return getComparatorFor(metadata, superColumn != null); - } - - public static AbstractType getComparatorFor(CFMetaData metadata, boolean subColumn) - { - return metadata.isSuper() - ? metadata.comparator.subtype(subColumn ? 1 : 0) - : metadata.comparator.asAbstractType(); - } - - // Extract the first component of a columnName, i.e. the super column name - public static ByteBuffer scName(Composite columnName) - { - return columnName.get(0); - } - - // Extract the 2nd component of a columnName, i.e. the sub-column name - public static ByteBuffer subName(Composite columnName) - { - return columnName.get(1); - } - - public static Composite startOf(ByteBuffer scName) - { - return CellNames.compositeDense(scName).start(); - } - - public static Composite endOf(ByteBuffer scName) - { - return CellNames.compositeDense(scName).end(); - } - - public static IDiskAtomFilter fromSCFilter(CellNameType type, ByteBuffer scName, IDiskAtomFilter filter) - { - if (filter instanceof NamesQueryFilter) - return fromSCNamesFilter(type, scName, (NamesQueryFilter)filter); - else - return fromSCSliceFilter(type, scName, (SliceQueryFilter)filter); - } - - public static IDiskAtomFilter fromSCNamesFilter(CellNameType type, ByteBuffer scName, NamesQueryFilter filter) - { - if (scName == null) - { - ColumnSlice[] slices = new ColumnSlice[filter.columns.size()]; - int i = 0; - for (CellName name : filter.columns) - { - // Note that, because the filter in argument is the one from thrift, 'name' are SimpleDenseCellName. - // So calling name.slice() would be incorrect, as simple cell names don't handle the EOC properly. - // This is why we call buffer() and rebuild a Composite of the right type before call slice(). - slices[i++] = type.make(name.toByteBuffer()).slice(); - } - return new SliceQueryFilter(slices, false, slices.length, 1); - } - else - { - SortedSet newColumns = new TreeSet<>(type); - for (CellName c : filter.columns) - newColumns.add(type.makeCellName(scName, c.toByteBuffer())); - return filter.withUpdatedColumns(newColumns); - } - } - - public static SliceQueryFilter fromSCSliceFilter(CellNameType type, ByteBuffer scName, SliceQueryFilter filter) - { - assert filter.slices.length == 1; - if (scName == null) - { - // The filter is on the super column name - CBuilder builder = type.builder(); - Composite start = filter.start().isEmpty() - ? Composites.EMPTY - : builder.buildWith(filter.start().toByteBuffer()).withEOC(filter.reversed ? Composite.EOC.END : Composite.EOC.START); - Composite finish = filter.finish().isEmpty() - ? Composites.EMPTY - : builder.buildWith(filter.finish().toByteBuffer()).withEOC(filter.reversed ? Composite.EOC.START : Composite.EOC.END); - return new SliceQueryFilter(start, finish, filter.reversed, filter.count, 1); - } - else - { - CBuilder builder = type.builder().add(scName); - Composite start = filter.start().isEmpty() - ? builder.build().withEOC(filter.reversed ? Composite.EOC.END : Composite.EOC.START) - : builder.buildWith(filter.start().toByteBuffer()); - Composite end = filter.finish().isEmpty() - ? builder.build().withEOC(filter.reversed ? Composite.EOC.START : Composite.EOC.END) - : builder.buildWith(filter.finish().toByteBuffer()); - return new SliceQueryFilter(start, end, filter.reversed, filter.count); - } - } -} diff --git a/src/java/org/apache/cassandra/db/SystemKeyspace.java b/src/java/org/apache/cassandra/db/SystemKeyspace.java index 4bc15224b6..34c617f5c3 100644 --- a/src/java/org/apache/cassandra/db/SystemKeyspace.java +++ b/src/java/org/apache/cassandra/db/SystemKeyspace.java @@ -36,11 +36,10 @@ import org.apache.cassandra.config.DatabaseDescriptor; import org.apache.cassandra.config.KSMetaData; import org.apache.cassandra.cql3.QueryProcessor; import org.apache.cassandra.cql3.UntypedResultSet; +import org.apache.cassandra.db.partitions.*; import org.apache.cassandra.db.commitlog.ReplayPosition; import org.apache.cassandra.db.compaction.CompactionHistoryTabularData; import org.apache.cassandra.db.compaction.LeveledCompactionStrategy; -import org.apache.cassandra.db.composites.Composite; -import org.apache.cassandra.db.filter.QueryFilter; import org.apache.cassandra.db.marshal.*; import org.apache.cassandra.dht.IPartitioner; import org.apache.cassandra.dht.Range; @@ -126,8 +125,10 @@ public final class SystemKeyspace + "in_progress_ballot timeuuid," + "most_recent_commit blob," + "most_recent_commit_at timeuuid," + + "most_recent_commit_version int," + "proposal blob," + "proposal_ballot timeuuid," + + "proposal_version int," + "PRIMARY KEY ((row_key), cf_id))") .compactionStrategyClass(LeveledCompactionStrategy.class); @@ -831,27 +832,22 @@ public final class SystemKeyspace public static boolean isIndexBuilt(String keyspaceName, String indexName) { - ColumnFamilyStore cfs = Keyspace.open(NAME).getColumnFamilyStore(BUILT_INDEXES); - QueryFilter filter = QueryFilter.getNamesFilter(decorate(ByteBufferUtil.bytes(keyspaceName)), - BUILT_INDEXES, - FBUtilities.singleton(cfs.getComparator().makeCellName(indexName), cfs.getComparator()), - System.currentTimeMillis()); - return ColumnFamilyStore.removeDeleted(cfs.getColumnFamily(filter), Integer.MAX_VALUE) != null; + String req = "SELECT index_name FROM %s.\"%s\" WHERE table_name=? AND index_name=?"; + UntypedResultSet result = executeInternal(String.format(req, NAME, BUILT_INDEXES), keyspaceName, indexName); + return !result.isEmpty(); } public static void setIndexBuilt(String keyspaceName, String indexName) { - ColumnFamily cf = ArrayBackedSortedColumns.factory.create(NAME, BUILT_INDEXES); - cf.addColumn(new BufferCell(cf.getComparator().makeCellName(indexName), ByteBufferUtil.EMPTY_BYTE_BUFFER, FBUtilities.timestampMicros())); - new Mutation(NAME, ByteBufferUtil.bytes(keyspaceName), cf).apply(); + String req = "INSERT INTO %s.\"%s\" (table_name, index_name) VALUES (?, ?)"; + executeInternal(String.format(req, NAME, BUILT_INDEXES), keyspaceName, indexName); forceBlockingFlush(BUILT_INDEXES); } public static void setIndexRemoved(String keyspaceName, String indexName) { - Mutation mutation = new Mutation(NAME, ByteBufferUtil.bytes(keyspaceName)); - mutation.delete(BUILT_INDEXES, BuiltIndexes.comparator.makeCellName(indexName), FBUtilities.timestampMicros()); - mutation.apply(); + String req = "DELETE FROM %s.\"%s\" WHERE table_name = ? AND index_name = ?"; + executeInternal(String.format(req, NAME, BUILT_INDEXES), keyspaceName, indexName); forceBlockingFlush(BUILT_INDEXES); } @@ -884,23 +880,26 @@ public final class SystemKeyspace return hostId; } - public static PaxosState loadPaxosState(ByteBuffer key, CFMetaData metadata) + + public static PaxosState loadPaxosState(DecoratedKey key, CFMetaData metadata) { String req = "SELECT * FROM system.%s WHERE row_key = ? AND cf_id = ?"; - UntypedResultSet results = executeInternal(String.format(req, PAXOS), key, metadata.cfId); + UntypedResultSet results = executeInternal(String.format(req, PAXOS), key.getKey(), metadata.cfId); if (results.isEmpty()) return new PaxosState(key, metadata); UntypedResultSet.Row row = results.one(); Commit promised = row.has("in_progress_ballot") - ? new Commit(key, row.getUUID("in_progress_ballot"), ArrayBackedSortedColumns.factory.create(metadata)) + ? new Commit(row.getUUID("in_progress_ballot"), new PartitionUpdate(metadata, key, metadata.partitionColumns(), 1)) : Commit.emptyCommit(key, metadata); // either we have both a recently accepted ballot and update or we have neither + int proposalVersion = row.has("proposal_version") ? row.getInt("proposal_version") : MessagingService.VERSION_21; Commit accepted = row.has("proposal") - ? new Commit(key, row.getUUID("proposal_ballot"), ColumnFamily.fromBytes(row.getBytes("proposal"))) + ? new Commit(row.getUUID("proposal_ballot"), PartitionUpdate.fromBytes(row.getBytes("proposal"), proposalVersion, key)) : Commit.emptyCommit(key, metadata); // either most_recent_commit and most_recent_commit_at will both be set, or neither + int mostRecentVersion = row.has("most_recent_commit_version") ? row.getInt("most_recent_commit_version") : MessagingService.VERSION_21; Commit mostRecent = row.has("most_recent_commit") - ? new Commit(key, row.getUUID("most_recent_commit_at"), ColumnFamily.fromBytes(row.getBytes("most_recent_commit"))) + ? new Commit(row.getUUID("most_recent_commit_at"), PartitionUpdate.fromBytes(row.getBytes("most_recent_commit"), mostRecentVersion, key)) : Commit.emptyCommit(key, metadata); return new PaxosState(promised, accepted, mostRecent); } @@ -910,21 +909,22 @@ public final class SystemKeyspace String req = "UPDATE system.%s USING TIMESTAMP ? AND TTL ? SET in_progress_ballot = ? WHERE row_key = ? AND cf_id = ?"; executeInternal(String.format(req, PAXOS), UUIDGen.microsTimestamp(promise.ballot), - paxosTtl(promise.update.metadata), + paxosTtl(promise.update.metadata()), promise.ballot, - promise.key, - promise.update.id()); + promise.update.partitionKey().getKey(), + promise.update.metadata().cfId); } public static void savePaxosProposal(Commit proposal) { - executeInternal(String.format("UPDATE system.%s USING TIMESTAMP ? AND TTL ? SET proposal_ballot = ?, proposal = ? WHERE row_key = ? AND cf_id = ?", PAXOS), + executeInternal(String.format("UPDATE system.%s USING TIMESTAMP ? AND TTL ? SET proposal_ballot = ?, proposal = ?, proposal_version = ? WHERE row_key = ? AND cf_id = ?", PAXOS), UUIDGen.microsTimestamp(proposal.ballot), - paxosTtl(proposal.update.metadata), + paxosTtl(proposal.update.metadata()), proposal.ballot, - proposal.update.toBytes(), - proposal.key, - proposal.update.id()); + PartitionUpdate.toBytes(proposal.update, MessagingService.current_version), + MessagingService.current_version, + proposal.update.partitionKey().getKey(), + proposal.update.metadata().cfId); } private static int paxosTtl(CFMetaData metadata) @@ -937,14 +937,15 @@ public final class SystemKeyspace { // We always erase the last proposal (with the commit timestamp to no erase more recent proposal in case the commit is old) // even though that's really just an optimization since SP.beginAndRepairPaxos will exclude accepted proposal older than the mrc. - String cql = "UPDATE system.%s USING TIMESTAMP ? AND TTL ? SET proposal_ballot = null, proposal = null, most_recent_commit_at = ?, most_recent_commit = ? WHERE row_key = ? AND cf_id = ?"; + String cql = "UPDATE system.%s USING TIMESTAMP ? AND TTL ? SET proposal_ballot = null, proposal = null, most_recent_commit_at = ?, most_recent_commit = ?, most_recent_commit_version = ? WHERE row_key = ? AND cf_id = ?"; executeInternal(String.format(cql, PAXOS), UUIDGen.microsTimestamp(commit.ballot), - paxosTtl(commit.update.metadata), + paxosTtl(commit.update.metadata()), commit.ballot, - commit.update.toBytes(), - commit.key, - commit.update.id()); + PartitionUpdate.toBytes(commit.update, MessagingService.current_version), + MessagingService.current_version, + commit.update.partitionKey().getKey(), + commit.update.metadata().cfId); } /** @@ -998,24 +999,24 @@ public final class SystemKeyspace public static void updateSizeEstimates(String keyspace, String table, Map, Pair> estimates) { long timestamp = FBUtilities.timestampMicros(); - Mutation mutation = new Mutation(NAME, UTF8Type.instance.decompose(keyspace)); + DecoratedKey key = decorate(UTF8Type.instance.decompose(keyspace)); + PartitionUpdate update = new PartitionUpdate(SizeEstimates, key, SizeEstimates.partitionColumns(), estimates.size()); + Mutation mutation = new Mutation(update); // delete all previous values with a single range tombstone. - mutation.deleteRange(SIZE_ESTIMATES, - SizeEstimates.comparator.make(table).start(), - SizeEstimates.comparator.make(table).end(), - timestamp - 1); + int nowInSec = FBUtilities.nowInSeconds(); + update.addRangeTombstone(Slice.make(SizeEstimates.comparator, table), new SimpleDeletionTime(timestamp - 1, nowInSec)); // add a CQL row for each primary token range. - ColumnFamily cells = mutation.addOrGet(SizeEstimates); for (Map.Entry, Pair> entry : estimates.entrySet()) { Range range = entry.getKey(); Pair values = entry.getValue(); - Composite prefix = SizeEstimates.comparator.make(table, range.left.toString(), range.right.toString()); - CFRowAdder adder = new CFRowAdder(cells, prefix, timestamp); - adder.add("partitions_count", values.left) - .add("mean_partition_size", values.right); + new RowUpdateBuilder(SizeEstimates, timestamp, mutation) + .clustering(table, range.left.toString(), range.right.toString()) + .add("partitions_count", values.left) + .add("mean_partition_size", values.right) + .build(); } mutation.apply(); diff --git a/src/java/org/apache/cassandra/db/UnfilteredDeserializer.java b/src/java/org/apache/cassandra/db/UnfilteredDeserializer.java new file mode 100644 index 0000000000..a15fb61762 --- /dev/null +++ b/src/java/org/apache/cassandra/db/UnfilteredDeserializer.java @@ -0,0 +1,414 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.cassandra.db; + +import java.io.DataInput; +import java.io.IOException; + +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import org.apache.cassandra.config.CFMetaData; +import org.apache.cassandra.db.rows.*; +import org.apache.cassandra.net.MessagingService; + +/** + * Helper class to deserialize Unfiltered object from disk efficiently. + * + * More precisely, this class is used by the low-level reader to ensure + * we don't do more work than necessary (i.e. we don't allocate/deserialize + * objects for things we don't care about). + */ +public abstract class UnfilteredDeserializer +{ + private static final Logger logger = LoggerFactory.getLogger(UnfilteredDeserializer.class); + + protected final CFMetaData metadata; + protected final DataInput in; + protected final SerializationHelper helper; + + protected UnfilteredDeserializer(CFMetaData metadata, + DataInput in, + SerializationHelper helper) + { + this.metadata = metadata; + this.in = in; + this.helper = helper; + } + + public static UnfilteredDeserializer create(CFMetaData metadata, + DataInput in, + SerializationHeader header, + SerializationHelper helper, + DeletionTime partitionDeletion, + boolean readAllAsDynamic) + { + if (helper.version >= MessagingService.VERSION_30) + return new CurrentDeserializer(metadata, in, header, helper); + else + return new OldFormatDeserializer(metadata, in, helper, partitionDeletion, readAllAsDynamic); + } + + /** + * Whether or not there is more atom to read. + */ + public abstract boolean hasNext() throws IOException; + + /** + * Compare the provided bound to the next atom to read on disk. + * + * This will not read/deserialize the whole atom but only what is necessary for the + * comparison. Whenever we know what to do with this atom (read it or skip it), + * readNext or skipNext should be called. + */ + public abstract int compareNextTo(Slice.Bound bound) throws IOException; + + /** + * Returns whether the next atom is a row or not. + */ + public abstract boolean nextIsRow() throws IOException; + + /** + * Returns whether the next atom is the static row or not. + */ + public abstract boolean nextIsStatic() throws IOException; + + /** + * Returns the next atom. + */ + public abstract Unfiltered readNext() throws IOException; + + /** + * Clears any state in this deserializer. + */ + public abstract void clearState() throws IOException; + + /** + * Skips the next atom. + */ + public abstract void skipNext() throws IOException; + + private static class CurrentDeserializer extends UnfilteredDeserializer + { + private final ClusteringPrefix.Deserializer clusteringDeserializer; + private final SerializationHeader header; + + private int nextFlags; + private boolean isReady; + private boolean isDone; + + private final ReusableRow row; + private final RangeTombstoneMarker.Builder markerBuilder; + + private CurrentDeserializer(CFMetaData metadata, + DataInput in, + SerializationHeader header, + SerializationHelper helper) + { + super(metadata, in, helper); + this.header = header; + this.clusteringDeserializer = new ClusteringPrefix.Deserializer(metadata.comparator, in, header); + this.row = new ReusableRow(metadata.clusteringColumns().size(), header.columns().regulars, true, metadata.isCounter()); + this.markerBuilder = new RangeTombstoneMarker.Builder(metadata.clusteringColumns().size()); + } + + public boolean hasNext() throws IOException + { + if (isReady) + return true; + + prepareNext(); + return !isDone; + } + + private void prepareNext() throws IOException + { + if (isDone) + return; + + nextFlags = in.readUnsignedByte(); + if (UnfilteredSerializer.isEndOfPartition(nextFlags)) + { + isDone = true; + isReady = false; + return; + } + + clusteringDeserializer.prepare(nextFlags); + isReady = true; + } + + public int compareNextTo(Slice.Bound bound) throws IOException + { + if (!isReady) + prepareNext(); + + assert !isDone; + + return clusteringDeserializer.compareNextTo(bound); + } + + public boolean nextIsRow() throws IOException + { + if (!isReady) + prepareNext(); + + return UnfilteredSerializer.kind(nextFlags) == Unfiltered.Kind.ROW; + } + + public boolean nextIsStatic() throws IOException + { + // This exists only for the sake of the OldFormatDeserializer + throw new UnsupportedOperationException(); + } + + public Unfiltered readNext() throws IOException + { + isReady = false; + if (UnfilteredSerializer.kind(nextFlags) == Unfiltered.Kind.RANGE_TOMBSTONE_MARKER) + { + markerBuilder.reset(); + RangeTombstone.Bound.Kind kind = clusteringDeserializer.deserializeNextBound(markerBuilder); + UnfilteredSerializer.serializer.deserializeMarkerBody(in, header, kind.isBoundary(), markerBuilder); + return markerBuilder.build(); + } + else + { + Row.Writer writer = row.writer(); + clusteringDeserializer.deserializeNextClustering(writer); + UnfilteredSerializer.serializer.deserializeRowBody(in, header, helper, nextFlags, writer); + return row; + } + } + + public void skipNext() throws IOException + { + isReady = false; + ClusteringPrefix.Kind kind = clusteringDeserializer.skipNext(); + if (UnfilteredSerializer.kind(nextFlags) == Unfiltered.Kind.RANGE_TOMBSTONE_MARKER) + { + UnfilteredSerializer.serializer.skipMarkerBody(in, header, kind.isBoundary()); + } + else + { + UnfilteredSerializer.serializer.skipRowBody(in, header, helper, nextFlags); + } + } + + public void clearState() + { + isReady = false; + isDone = false; + } + } + + public static class OldFormatDeserializer extends UnfilteredDeserializer + { + private final boolean readAllAsDynamic; + private boolean skipStatic; + + private int nextFlags; + private boolean isDone; + private boolean isStart = true; + + private final LegacyLayout.CellGrouper grouper; + private LegacyLayout.LegacyAtom nextAtom; + + private boolean staticFinished; + private LegacyLayout.LegacyAtom savedAtom; + + private final LegacyLayout.TombstoneTracker tombstoneTracker; + + private RangeTombstoneMarker closingMarker; + + private OldFormatDeserializer(CFMetaData metadata, + DataInput in, + SerializationHelper helper, + DeletionTime partitionDeletion, + boolean readAllAsDynamic) + { + super(metadata, in, helper); + this.readAllAsDynamic = readAllAsDynamic; + this.grouper = new LegacyLayout.CellGrouper(metadata, helper); + this.tombstoneTracker = new LegacyLayout.TombstoneTracker(metadata, partitionDeletion); + } + + public void setSkipStatic() + { + this.skipStatic = true; + } + + public boolean hasNext() throws IOException + { + if (nextAtom != null) + return true; + + if (isDone) + return false; + + return deserializeNextAtom(); + } + + private boolean deserializeNextAtom() throws IOException + { + if (staticFinished && savedAtom != null) + { + nextAtom = savedAtom; + savedAtom = null; + return true; + } + + while (true) + { + nextAtom = LegacyLayout.readLegacyAtom(metadata, in, readAllAsDynamic); + if (nextAtom == null) + { + isDone = true; + return false; + } + else if (tombstoneTracker.isShadowed(nextAtom)) + { + // We don't want to return shadowed data because that would fail the contract + // of UnfilteredRowIterator. However the old format could have shadowed data, so filter it here. + nextAtom = null; + continue; + } + + tombstoneTracker.update(nextAtom); + + // For static compact tables, the "column_metadata" columns are supposed to be static, but in the old + // format they are intermingled with other columns. We deal with that with 2 different strategy: + // 1) for thrift queries, we basically consider everything as a "dynamic" cell. This is ok because + // that's basically what we end up with on ThriftResultsMerger has done its thing. + // 2) otherwise, we make sure to extract the "static" columns first (see AbstractSSTableIterator.readStaticRow + // and SSTableSimpleIterator.readStaticRow) as a first pass. So, when we do a 2nd pass for dynamic columns + // (which in practice we only do for compactions), we want to ignore those extracted static columns. + if (skipStatic && metadata.isStaticCompactTable() && nextAtom.isCell()) + { + LegacyLayout.LegacyCell cell = nextAtom.asCell(); + if (cell.name.column.isStatic()) + { + nextAtom = null; + continue; + } + } + + // We want to fetch the static row as the first thing this deserializer return. + // However, in practice, it's possible to have range tombstone before the static row cells + // if that tombstone has an empty start. So if we do, we save it initially so we can get + // to the static parts (if there is any). + if (isStart) + { + isStart = false; + if (!nextAtom.isCell()) + { + LegacyLayout.LegacyRangeTombstone tombstone = nextAtom.asRangeTombstone(); + if (tombstone.start.bound.size() == 0) + { + savedAtom = tombstone; + nextAtom = LegacyLayout.readLegacyAtom(metadata, in, readAllAsDynamic); + if (nextAtom == null) + { + // That was actually the only atom so use it after all + nextAtom = savedAtom; + savedAtom = null; + } + else if (!nextAtom.isStatic()) + { + // We don't have anything static. So we do want to send first + // the saved atom, so switch + LegacyLayout.LegacyAtom atom = nextAtom; + nextAtom = savedAtom; + savedAtom = atom; + } + } + } + } + + return true; + } + } + + private void checkReady() throws IOException + { + if (nextAtom == null) + hasNext(); + assert !isDone; + } + + public int compareNextTo(Slice.Bound bound) throws IOException + { + checkReady(); + return metadata.comparator.compare(nextAtom, bound); + } + + public boolean nextIsRow() throws IOException + { + checkReady(); + if (nextAtom.isCell()) + return true; + + LegacyLayout.LegacyRangeTombstone tombstone = nextAtom.asRangeTombstone(); + return tombstone.isCollectionTombstone() || tombstone.isRowDeletion(metadata); + } + + public boolean nextIsStatic() throws IOException + { + checkReady(); + return nextAtom.isStatic(); + } + + public Unfiltered readNext() throws IOException + { + if (!nextIsRow()) + { + LegacyLayout.LegacyRangeTombstone tombstone = nextAtom.asRangeTombstone(); + // TODO: this is actually more complex, we can have repeated markers etc.... + if (closingMarker == null) + throw new UnsupportedOperationException(); + closingMarker = new RangeTombstoneBoundMarker(tombstone.stop.bound, tombstone.deletionTime); + return new RangeTombstoneBoundMarker(tombstone.start.bound, tombstone.deletionTime); + } + + LegacyLayout.CellGrouper grouper = nextAtom.isStatic() + ? LegacyLayout.CellGrouper.staticGrouper(metadata, helper) + : this.grouper; + + grouper.reset(); + grouper.addAtom(nextAtom); + while (deserializeNextAtom() && grouper.addAtom(nextAtom)) + { + } + + // if this was the first static row, we're done with it. Otherwise, we're also done with static. + staticFinished = true; + return grouper.getRow(); + } + + public void skipNext() throws IOException + { + readNext(); + } + + public void clearState() + { + isDone = false; + nextAtom = null; + } + } +} diff --git a/src/java/org/apache/cassandra/db/ColumnFamilyType.java b/src/java/org/apache/cassandra/db/UnknownColumnException.java similarity index 51% rename from src/java/org/apache/cassandra/db/ColumnFamilyType.java rename to src/java/org/apache/cassandra/db/UnknownColumnException.java index 51e8b63295..55dc453865 100644 --- a/src/java/org/apache/cassandra/db/ColumnFamilyType.java +++ b/src/java/org/apache/cassandra/db/UnknownColumnException.java @@ -17,24 +17,35 @@ */ package org.apache.cassandra.db; -/** - * column family type enum - */ -public enum ColumnFamilyType -{ - Standard, - Super; +import java.nio.ByteBuffer; - public static ColumnFamilyType create(String name) +import org.apache.cassandra.config.CFMetaData; +import org.apache.cassandra.db.marshal.UTF8Type; +import org.apache.cassandra.utils.ByteBufferUtil; + +/** + * Exception thrown when we read a column internally that is unknown. Note that + * this is an internal exception and is not meant to be user facing. + */ +public class UnknownColumnException extends Exception +{ + public final ByteBuffer columnName; + + public UnknownColumnException(CFMetaData metadata, ByteBuffer columnName) + { + super(String.format("Unknown column %s in table %s.%s", stringify(columnName), metadata.ksName, metadata.cfName)); + this.columnName = columnName; + } + + private static String stringify(ByteBuffer name) { try { - // TODO thrift optional parameter in CfDef is leaking down here which it shouldn't - return name == null ? ColumnFamilyType.Standard : ColumnFamilyType.valueOf(name); + return UTF8Type.instance.getString(name); } - catch (IllegalArgumentException e) + catch (Exception e) { - return null; + return ByteBufferUtil.bytesToHex(name); } } } diff --git a/src/java/org/apache/cassandra/db/columniterator/AbstractSSTableIterator.java b/src/java/org/apache/cassandra/db/columniterator/AbstractSSTableIterator.java new file mode 100644 index 0000000000..b4062510a4 --- /dev/null +++ b/src/java/org/apache/cassandra/db/columniterator/AbstractSSTableIterator.java @@ -0,0 +1,424 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.cassandra.db.columniterator; + +import java.io.IOException; +import java.util.Collections; +import java.util.Iterator; +import java.util.List; + +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import org.apache.cassandra.config.CFMetaData; +import org.apache.cassandra.db.*; +import org.apache.cassandra.db.filter.ColumnFilter; +import org.apache.cassandra.db.rows.*; +import org.apache.cassandra.io.sstable.format.SSTableReader; +import org.apache.cassandra.io.sstable.CorruptSSTableException; +import org.apache.cassandra.io.sstable.IndexHelper; +import org.apache.cassandra.io.util.FileDataInput; +import org.apache.cassandra.io.util.FileMark; +import org.apache.cassandra.utils.ByteBufferUtil; + +abstract class AbstractSSTableIterator implements SliceableUnfilteredRowIterator +{ + private static final Logger logger = LoggerFactory.getLogger(AbstractSSTableIterator.class); + + protected final SSTableReader sstable; + protected final DecoratedKey key; + protected final DeletionTime partitionLevelDeletion; + protected final ColumnFilter columns; + protected final SerializationHelper helper; + + protected final Row staticRow; + protected final Reader reader; + + private final boolean isForThrift; + + private boolean isClosed; + + @SuppressWarnings("resource") // We need this because the analysis is not able to determine that we do close + // file on every path where we created it. + protected AbstractSSTableIterator(SSTableReader sstable, + FileDataInput file, + DecoratedKey key, + RowIndexEntry indexEntry, + ColumnFilter columnFilter, + boolean isForThrift) + { + this.sstable = sstable; + this.key = key; + this.columns = columnFilter; + this.helper = new SerializationHelper(sstable.descriptor.version.correspondingMessagingVersion(), SerializationHelper.Flag.LOCAL, columnFilter); + this.isForThrift = isForThrift; + + if (indexEntry == null) + { + this.partitionLevelDeletion = DeletionTime.LIVE; + this.reader = null; + this.staticRow = Rows.EMPTY_STATIC_ROW; + } + else + { + boolean shouldCloseFile = file == null; + try + { + // We seek to the beginning to the partition if either: + // - the partition is not indexed; we then have a single block to read anyway + // and we need to read the partition deletion time. + // - we're querying static columns. + boolean needSeekAtPartitionStart = !indexEntry.isIndexed() || !columns.fetchedColumns().statics.isEmpty(); + + // For CQL queries on static compact tables, we only want to consider static value (only those are exposed), + // but readStaticRow have already read them and might in fact have consumed the whole partition (when reading + // the legacy file format), so set the reader to null so we don't try to read anything more. We can remove this + // once we drop support for the legacy file format + boolean needsReader = sstable.descriptor.version.storeRows() || isForThrift || !sstable.metadata.isStaticCompactTable(); + + if (needSeekAtPartitionStart) + { + // Not indexed (or is reading static), set to the beginning of the partition and read partition level deletion there + if (file == null) + file = sstable.getFileDataInput(indexEntry.position); + else + file.seek(indexEntry.position); + + ByteBufferUtil.skipShortLength(file); // Skip partition key + this.partitionLevelDeletion = DeletionTime.serializer.deserialize(file); + + // Note that this needs to be called after file != null and after the partitionDeletion has been set, but before readStaticRow + // (since it uses it) so we can't move that up (but we'll be able to simplify as soon as we drop support for the old file format). + this.reader = needsReader ? createReader(indexEntry, file, needSeekAtPartitionStart, shouldCloseFile) : null; + this.staticRow = readStaticRow(sstable, file, helper, columns.fetchedColumns().statics, isForThrift, reader == null ? null : reader.deserializer); + } + else + { + this.partitionLevelDeletion = indexEntry.deletionTime(); + this.staticRow = Rows.EMPTY_STATIC_ROW; + this.reader = needsReader ? createReader(indexEntry, file, needSeekAtPartitionStart, shouldCloseFile) : null; + } + + if (reader == null && shouldCloseFile) + file.close(); + } + catch (IOException e) + { + sstable.markSuspect(); + String filePath = file.getPath(); + if (shouldCloseFile && file != null) + { + try + { + file.close(); + } + catch (IOException suppressed) + { + e.addSuppressed(suppressed); + } + } + throw new CorruptSSTableException(e, filePath); + } + } + } + + private static Row readStaticRow(SSTableReader sstable, + FileDataInput file, + SerializationHelper helper, + Columns statics, + boolean isForThrift, + UnfilteredDeserializer deserializer) throws IOException + { + if (!sstable.descriptor.version.storeRows()) + { + if (!sstable.metadata.isCompactTable()) + { + assert deserializer != null; + return deserializer.hasNext() && deserializer.nextIsStatic() + ? (Row)deserializer.readNext() + : Rows.EMPTY_STATIC_ROW; + } + + // For compact tables, we use statics for the "column_metadata" definition. However, in the old format, those + // "column_metadata" are intermingled as any other "cell". In theory, this means that we'd have to do a first + // pass to extract the static values. However, for thrift, we'll use the ThriftResultsMerger right away which + // will re-merge static values with dynamic ones, so we can just ignore static and read every cell as a + // "dynamic" one. For CQL, if the table is a "static compact", then is has only static columns exposed and no + // dynamic ones. So we do a pass to extract static columns here, but will have no more work to do. Otherwise, + // the table won't have static columns. + if (statics.isEmpty() || isForThrift) + return Rows.EMPTY_STATIC_ROW; + + assert sstable.metadata.isStaticCompactTable() && !isForThrift; + + // As said above, if it's a CQL query and the table is a "static compact", the only exposed columns are the + // static ones. So we don't have to mark the position to seek back later. + return LegacyLayout.extractStaticColumns(sstable.metadata, file, statics); + } + + if (!sstable.header.hasStatic()) + return Rows.EMPTY_STATIC_ROW; + + if (statics.isEmpty()) + { + UnfilteredSerializer.serializer.skipStaticRow(file, sstable.header, helper); + return Rows.EMPTY_STATIC_ROW; + } + else + { + return UnfilteredSerializer.serializer.deserializeStaticRow(file, sstable.header, helper); + } + } + + protected abstract Reader createReader(RowIndexEntry indexEntry, FileDataInput file, boolean isAtPartitionStart, boolean shouldCloseFile); + + public CFMetaData metadata() + { + return sstable.metadata; + } + + public PartitionColumns columns() + { + return columns.fetchedColumns(); + } + + public DecoratedKey partitionKey() + { + return key; + } + + public DeletionTime partitionLevelDeletion() + { + return partitionLevelDeletion; + } + + public Row staticRow() + { + return staticRow; + } + + public RowStats stats() + { + // We could return sstable.header.stats(), but this may not be as accurate than the actual sstable stats (see + // SerializationHeader.make() for details) so we use the latter instead. + return new RowStats(sstable.getMinTimestamp(), sstable.getMinLocalDeletionTime(), sstable.getMinTTL(), sstable.getAvgColumnSetPerRow()); + } + + public boolean hasNext() + { + try + { + return reader != null && reader.hasNext(); + } + catch (IOException e) + { + try + { + closeInternal(); + } + catch (IOException suppressed) + { + e.addSuppressed(suppressed); + } + sstable.markSuspect(); + throw new CorruptSSTableException(e, reader.file.getPath()); + } + } + + public Unfiltered next() + { + try + { + assert reader != null; + return reader.next(); + } + catch (IOException e) + { + try + { + closeInternal(); + } + catch (IOException suppressed) + { + e.addSuppressed(suppressed); + } + sstable.markSuspect(); + throw new CorruptSSTableException(e, reader.file.getPath()); + } + } + + public Iterator slice(Slice slice) + { + try + { + if (reader == null) + return Collections.emptyIterator(); + + return reader.slice(slice); + } + catch (IOException e) + { + try + { + closeInternal(); + } + catch (IOException suppressed) + { + e.addSuppressed(suppressed); + } + sstable.markSuspect(); + throw new CorruptSSTableException(e, reader.file.getPath()); + } + } + + public void remove() + { + throw new UnsupportedOperationException(); + } + + private void closeInternal() throws IOException + { + // It's important to make closing idempotent since it would bad to double-close 'file' as its a RandomAccessReader + // and its close is not idemptotent in the case where we recycle it. + if (isClosed) + return; + + if (reader != null) + reader.close(); + + isClosed = true; + } + + public void close() + { + try + { + closeInternal(); + } + catch (IOException e) + { + sstable.markSuspect(); + throw new CorruptSSTableException(e, reader.file.getPath()); + } + } + + protected abstract class Reader + { + private final boolean shouldCloseFile; + public FileDataInput file; + + protected UnfilteredDeserializer deserializer; + + // Records the currently open range tombstone (if any) + protected DeletionTime openMarker = null; + + protected Reader(FileDataInput file, boolean shouldCloseFile) + { + this.file = file; + this.shouldCloseFile = shouldCloseFile; + if (file != null) + createDeserializer(); + } + + private void createDeserializer() + { + assert file != null && deserializer == null; + deserializer = UnfilteredDeserializer.create(sstable.metadata, file, sstable.header, helper, partitionLevelDeletion, isForThrift); + } + + protected void seekToPosition(long position) throws IOException + { + // This may be the first time we're actually looking into the file + if (file == null) + { + file = sstable.getFileDataInput(position); + createDeserializer(); + } + else + { + file.seek(position); + deserializer.clearState(); + } + } + + protected void updateOpenMarker(RangeTombstoneMarker marker) + { + // Note that we always read index blocks in forward order so this method is always called in forward order + openMarker = marker.isOpen(false) ? marker.openDeletionTime(false) : null; + } + + protected DeletionTime getAndClearOpenMarker() + { + DeletionTime toReturn = openMarker; + openMarker = null; + return toReturn; + } + + public abstract boolean hasNext() throws IOException; + public abstract Unfiltered next() throws IOException; + public abstract Iterator slice(Slice slice) throws IOException; + + public void close() throws IOException + { + if (shouldCloseFile && file != null) + file.close(); + } + } + + protected abstract class IndexedReader extends Reader + { + protected final RowIndexEntry indexEntry; + protected final List indexes; + + protected int currentIndexIdx = -1; + + // Marks the beginning of the block corresponding to currentIndexIdx. + protected FileMark mark; + + // !isInit means we have never seeked in the file and thus shouldn't read as we could be anywhere + protected boolean isInit; + + protected IndexedReader(FileDataInput file, boolean shouldCloseFile, RowIndexEntry indexEntry, boolean isInit) + { + super(file, shouldCloseFile); + this.indexEntry = indexEntry; + this.indexes = indexEntry.columnsIndex(); + this.isInit = isInit; + } + + // Should be called when we're at the beginning of blockIdx. + protected void updateBlock(int blockIdx) throws IOException + { + seekToPosition(indexEntry.position + indexes.get(blockIdx).offset); + + currentIndexIdx = blockIdx; + openMarker = blockIdx > 0 ? indexes.get(blockIdx - 1).endOpenMarker : null; + mark = file.mark(); + } + + public IndexHelper.IndexInfo currentIndex() + { + return indexes.get(currentIndexIdx); + } + + public IndexHelper.IndexInfo previousIndex() + { + return currentIndexIdx <= 1 ? null : indexes.get(currentIndexIdx - 1); + } + } +} diff --git a/src/java/org/apache/cassandra/db/columniterator/IColumnIteratorFactory.java b/src/java/org/apache/cassandra/db/columniterator/IColumnIteratorFactory.java deleted file mode 100644 index 46983e9787..0000000000 --- a/src/java/org/apache/cassandra/db/columniterator/IColumnIteratorFactory.java +++ /dev/null @@ -1,44 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.apache.cassandra.db.columniterator; -/* - * - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - * - */ - - -public interface IColumnIteratorFactory -{ - OnDiskAtomIterator create(); -} diff --git a/src/java/org/apache/cassandra/db/columniterator/LazyColumnIterator.java b/src/java/org/apache/cassandra/db/columniterator/LazyColumnIterator.java deleted file mode 100644 index 9d1cecbe56..0000000000 --- a/src/java/org/apache/cassandra/db/columniterator/LazyColumnIterator.java +++ /dev/null @@ -1,100 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.apache.cassandra.db.columniterator; -/* - * - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - * - */ - - -import com.google.common.collect.AbstractIterator; -import org.apache.cassandra.db.ColumnFamily; -import org.apache.cassandra.db.DecoratedKey; -import org.apache.cassandra.db.OnDiskAtom; - -import java.io.IOException; - - -/* - * The goal of this encapsulating OnDiskAtomIterator is to delay the use of - * the filter until columns are actually queried. - * The reason for that is get_paged_slice because it change the start of - * the filter after having seen the first row, and so we must not use the - * filter before the row data is actually queried. However, mergeIterator - * needs to "fetch" a row in advance. But all it needs is the key and so - * this IColumnIterator make sure getKey() can be called without triggering - * the use of the filter itself. - */ -public class LazyColumnIterator extends AbstractIterator implements OnDiskAtomIterator -{ - private final DecoratedKey key; - private final IColumnIteratorFactory subIteratorFactory; - - private OnDiskAtomIterator subIterator; - - public LazyColumnIterator(DecoratedKey key, IColumnIteratorFactory subIteratorFactory) - { - this.key = key; - this.subIteratorFactory = subIteratorFactory; - } - - private OnDiskAtomIterator getSubIterator() - { - if (subIterator == null) - subIterator = subIteratorFactory.create(); - return subIterator; - } - - protected OnDiskAtom computeNext() - { - getSubIterator(); - return subIterator.hasNext() ? subIterator.next() : endOfData(); - } - - public ColumnFamily getColumnFamily() - { - return getSubIterator().getColumnFamily(); - } - - public DecoratedKey getKey() - { - return key; - } - - public void close() throws IOException - { - if (subIterator != null) - subIterator.close(); - } -} diff --git a/src/java/org/apache/cassandra/db/columniterator/OnDiskAtomIterator.java b/src/java/org/apache/cassandra/db/columniterator/OnDiskAtomIterator.java deleted file mode 100644 index 21c38f7f59..0000000000 --- a/src/java/org/apache/cassandra/db/columniterator/OnDiskAtomIterator.java +++ /dev/null @@ -1,42 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.apache.cassandra.db.columniterator; - -import java.io.IOException; -import org.apache.cassandra.db.ColumnFamily; -import org.apache.cassandra.db.DecoratedKey; -import org.apache.cassandra.db.OnDiskAtom; -import org.apache.cassandra.utils.CloseableIterator; - -public interface OnDiskAtomIterator extends CloseableIterator -{ - /** - * @return A ColumnFamily holding metadata for the row being iterated. - * Do not modify this CF. Whether it is empty or not is implementation-dependent. - */ - public abstract ColumnFamily getColumnFamily(); - - /** - * @return the current row key - */ - public DecoratedKey getKey(); - - /** clean up any open resources */ - public void close() throws IOException; -} - diff --git a/src/java/org/apache/cassandra/db/columniterator/SSTableIterator.java b/src/java/org/apache/cassandra/db/columniterator/SSTableIterator.java new file mode 100644 index 0000000000..4fd5205db4 --- /dev/null +++ b/src/java/org/apache/cassandra/db/columniterator/SSTableIterator.java @@ -0,0 +1,292 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.cassandra.db.columniterator; + +import java.io.IOException; +import java.util.Collections; +import java.util.Iterator; +import java.util.List; + +import com.google.common.collect.AbstractIterator; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import org.apache.cassandra.db.*; +import org.apache.cassandra.db.filter.ColumnFilter; +import org.apache.cassandra.db.rows.*; +import org.apache.cassandra.io.sstable.format.SSTableReader; +import org.apache.cassandra.io.sstable.CorruptSSTableException; +import org.apache.cassandra.io.sstable.IndexHelper; +import org.apache.cassandra.io.util.FileDataInput; +import org.apache.cassandra.utils.ByteBufferUtil; + +/** + * A Cell Iterator over SSTable + */ +public class SSTableIterator extends AbstractSSTableIterator +{ + private static final Logger logger = LoggerFactory.getLogger(SSTableIterator.class); + + public SSTableIterator(SSTableReader sstable, DecoratedKey key, ColumnFilter columns, boolean isForThrift) + { + this(sstable, null, key, sstable.getPosition(key, SSTableReader.Operator.EQ), columns, isForThrift); + } + + public SSTableIterator(SSTableReader sstable, + FileDataInput file, + DecoratedKey key, + RowIndexEntry indexEntry, + ColumnFilter columns, + boolean isForThrift) + { + super(sstable, file, key, indexEntry, columns, isForThrift); + } + + protected Reader createReader(RowIndexEntry indexEntry, FileDataInput file, boolean isAtPartitionStart, boolean shouldCloseFile) + { + return indexEntry.isIndexed() + ? new ForwardIndexedReader(indexEntry, file, isAtPartitionStart, shouldCloseFile) + : new ForwardReader(file, isAtPartitionStart, shouldCloseFile); + } + + public boolean isReverseOrder() + { + return false; + } + + private class ForwardReader extends Reader + { + private ForwardReader(FileDataInput file, boolean isAtPartitionStart, boolean shouldCloseFile) + { + super(file, shouldCloseFile); + assert isAtPartitionStart; + } + + public boolean hasNext() throws IOException + { + assert deserializer != null; + return deserializer.hasNext(); + } + + public Unfiltered next() throws IOException + { + return deserializer.readNext(); + } + + public Iterator slice(final Slice slice) throws IOException + { + return new AbstractIterator() + { + private boolean beforeStart = true; + + protected Unfiltered computeNext() + { + try + { + // While we're before the start of the slice, we can skip row but we should keep + // track of open range tombstones + if (beforeStart) + { + // Note that the following comparison is not strict. The reason is that the only cases + // where it can be == is if the "next" is a RT start marker (either a '[' of a ')[' boundary), + // and if we had a strict inequality and an open RT marker before this, we would issue + // the open marker first, and then return then next later, which would yet in the + // stream both '[' (or '(') and then ')[' for the same clustering value, which is wrong. + // By using a non-strict inequality, we avoid that problem (if we do get ')[' for the same + // clustering value than the slice, we'll simply record it in 'openMarker'). + while (deserializer.hasNext() && deserializer.compareNextTo(slice.start()) <= 0) + { + if (deserializer.nextIsRow()) + deserializer.skipNext(); + else + updateOpenMarker((RangeTombstoneMarker)deserializer.readNext()); + } + + beforeStart = false; + + // We've reached the beginning of our queried slice. If we have an open marker + // we should return that first. + if (openMarker != null) + return new RangeTombstoneBoundMarker(slice.start(), openMarker); + } + + if (deserializer.hasNext() && deserializer.compareNextTo(slice.end()) <= 0) + { + Unfiltered next = deserializer.readNext(); + if (next.kind() == Unfiltered.Kind.RANGE_TOMBSTONE_MARKER) + updateOpenMarker((RangeTombstoneMarker)next); + return next; + } + + // If we have an open marker, we should close it before finishing + if (openMarker != null) + return new RangeTombstoneBoundMarker(slice.end(), getAndClearOpenMarker()); + + return endOfData(); + } + catch (IOException e) + { + try + { + close(); + } + catch (IOException suppressed) + { + e.addSuppressed(suppressed); + } + sstable.markSuspect(); + throw new CorruptSSTableException(e, file.getPath()); + } + } + }; + } + } + + private class ForwardIndexedReader extends IndexedReader + { + private ForwardIndexedReader(RowIndexEntry indexEntry, FileDataInput file, boolean isAtPartitionStart, boolean shouldCloseFile) + { + super(file, shouldCloseFile, indexEntry, isAtPartitionStart); + } + + public boolean hasNext() throws IOException + { + // If it's called before we've created the file, create it. This then mean + // we're reading from the beginning of the partition. + if (!isInit) + { + seekToPosition(indexEntry.position); + ByteBufferUtil.skipShortLength(file); // partition key + DeletionTime.serializer.skip(file); // partition deletion + if (sstable.header.hasStatic()) + UnfilteredSerializer.serializer.skipStaticRow(file, sstable.header, helper); + isInit = true; + } + return deserializer.hasNext(); + } + + public Unfiltered next() throws IOException + { + return deserializer.readNext(); + } + + public Iterator slice(final Slice slice) throws IOException + { + final List indexes = indexEntry.columnsIndex(); + + // if our previous slicing already got us the biggest row in the sstable, we're done + if (currentIndexIdx >= indexes.size()) + return Collections.emptyIterator(); + + // Find the first index block we'll need to read for the slice. + final int startIdx = IndexHelper.indexFor(slice.start(), indexes, sstable.metadata.comparator, false, currentIndexIdx); + if (startIdx >= indexes.size()) + return Collections.emptyIterator(); + + // If that's the last block we were reading, we're already where we want to be. Otherwise, + // seek to that first block + if (startIdx != currentIndexIdx) + updateBlock(startIdx); + + // Find the last index block we'll need to read for the slice. + final int endIdx = IndexHelper.indexFor(slice.end(), indexes, sstable.metadata.comparator, false, startIdx); + + final IndexHelper.IndexInfo startIndex = currentIndex(); + + // The index search is based on the last name of the index blocks, so at that point we have that: + // 1) indexes[startIdx - 1].lastName < slice.start <= indexes[startIdx].lastName + // 2) indexes[endIdx - 1].lastName < slice.end <= indexes[endIdx].lastName + // so if startIdx == endIdx and slice.end < indexes[startIdx].firstName, we're guaranteed that the + // whole slice is between the previous block end and this bloc start, and thus has no corresponding + // data. One exception is if the previous block ends with an openMarker as it will cover our slice + // and we need to return it. + if (startIdx == endIdx && metadata().comparator.compare(slice.end(), startIndex.firstName) < 0 && openMarker == null && sstable.descriptor.version.storeRows()) + return Collections.emptyIterator(); + + return new AbstractIterator() + { + private boolean beforeStart = true; + private int currentIndexIdx = startIdx; + + protected Unfiltered computeNext() + { + try + { + // While we're before the start of the slice, we can skip row but we should keep + // track of open range tombstones + if (beforeStart) + { + // See ForwardReader equivalent method to see why this inequality is not strict. + while (deserializer.hasNext() && deserializer.compareNextTo(slice.start()) <= 0) + { + if (deserializer.nextIsRow()) + deserializer.skipNext(); + else + updateOpenMarker((RangeTombstoneMarker)deserializer.readNext()); + } + + beforeStart = false; + + // We've reached the beginning of our queried slice. If we have an open marker + // we should return that first. + if (openMarker != null) + return new RangeTombstoneBoundMarker(slice.start(), openMarker); + } + + // If we've crossed an index block boundary, update our informations + if (currentIndexIdx < indexes.size() && file.bytesPastMark(mark) >= currentIndex().width) + updateBlock(++currentIndexIdx); + + // Return the next atom unless we've reached the end, or we're beyond our slice + // end (note that unless we're on the last block for the slice, there is no point + // in checking the slice end). + if (currentIndexIdx < indexes.size() + && currentIndexIdx <= endIdx + && deserializer.hasNext() + && (currentIndexIdx != endIdx || deserializer.compareNextTo(slice.end()) <= 0)) + { + Unfiltered next = deserializer.readNext(); + if (next.kind() == Unfiltered.Kind.RANGE_TOMBSTONE_MARKER) + updateOpenMarker((RangeTombstoneMarker)next); + return next; + } + + // If we have an open marker, we should close it before finishing + if (openMarker != null) + return new RangeTombstoneBoundMarker(slice.end(), getAndClearOpenMarker()); + + return endOfData(); + } + catch (IOException e) + { + try + { + close(); + } + catch (IOException suppressed) + { + e.addSuppressed(suppressed); + } + sstable.markSuspect(); + throw new CorruptSSTableException(e, file.getPath()); + } + } + }; + } + } +} diff --git a/src/java/org/apache/cassandra/db/columniterator/SSTableReversedIterator.java b/src/java/org/apache/cassandra/db/columniterator/SSTableReversedIterator.java new file mode 100644 index 0000000000..0e18d4a001 --- /dev/null +++ b/src/java/org/apache/cassandra/db/columniterator/SSTableReversedIterator.java @@ -0,0 +1,388 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.cassandra.db.columniterator; + +import java.io.IOException; +import java.util.*; + +import com.google.common.collect.AbstractIterator; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import org.apache.cassandra.config.CFMetaData; +import org.apache.cassandra.db.*; +import org.apache.cassandra.db.filter.ColumnFilter; +import org.apache.cassandra.db.rows.*; +import org.apache.cassandra.db.partitions.AbstractPartitionData; +import org.apache.cassandra.io.sstable.format.SSTableReader; +import org.apache.cassandra.io.sstable.CorruptSSTableException; +import org.apache.cassandra.io.sstable.IndexHelper; +import org.apache.cassandra.io.util.FileDataInput; +import org.apache.cassandra.io.util.FileMark; +import org.apache.cassandra.utils.ByteBufferUtil; + +/** + * A Cell Iterator in reversed clustering order over SSTable + */ +public class SSTableReversedIterator extends AbstractSSTableIterator +{ + private static final Logger logger = LoggerFactory.getLogger(SSTableReversedIterator.class); + + public SSTableReversedIterator(SSTableReader sstable, DecoratedKey key, ColumnFilter columns, boolean isForThrift) + { + this(sstable, null, key, sstable.getPosition(key, SSTableReader.Operator.EQ), columns, isForThrift); + } + + public SSTableReversedIterator(SSTableReader sstable, + FileDataInput file, + DecoratedKey key, + RowIndexEntry indexEntry, + ColumnFilter columns, + boolean isForThrift) + { + super(sstable, file, key, indexEntry, columns, isForThrift); + } + + protected Reader createReader(RowIndexEntry indexEntry, FileDataInput file, boolean isAtPartitionStart, boolean shouldCloseFile) + { + return indexEntry.isIndexed() + ? new ReverseIndexedReader(indexEntry, file, isAtPartitionStart, shouldCloseFile) + : new ReverseReader(file, isAtPartitionStart, shouldCloseFile); + } + + public boolean isReverseOrder() + { + return true; + } + + private ReusablePartitionData createBuffer(int blocksCount) + { + int estimatedRowCount = 16; + int columnCount = metadata().partitionColumns().regulars.columnCount(); + if (columnCount == 0 || metadata().clusteringColumns().size() == 0) + { + estimatedRowCount = 1; + } + else + { + try + { + // To avoid wasted resizing we guess-estimate the number of rows we're likely to read. For that + // we use the stats on the number of rows per partition for that sstable. + // FIXME: so far we only keep stats on cells, so to get a rough estimate on the number of rows, + // we divide by the number of regular columns the table has. We should fix once we collect the + // stats on rows + int estimatedRowsPerPartition = (int)(sstable.getEstimatedColumnCount().percentile(0.75) / columnCount); + estimatedRowCount = Math.max(estimatedRowsPerPartition / blocksCount, 1); + } + catch (IllegalStateException e) + { + // The EstimatedHistogram mean() method can throw this (if it overflows). While such overflow + // shouldn't happen, it's not worth taking the risk of letting the exception bubble up. + } + } + return new ReusablePartitionData(metadata(), partitionKey(), DeletionTime.LIVE, columns(), estimatedRowCount); + } + + private class ReverseReader extends Reader + { + private ReusablePartitionData partition; + private UnfilteredRowIterator iterator; + + private ReverseReader(FileDataInput file, boolean isAtPartitionStart, boolean shouldCloseFile) + { + super(file, shouldCloseFile); + assert isAtPartitionStart; + } + + public boolean hasNext() throws IOException + { + if (partition == null) + { + partition = createBuffer(1); + partition.populateFrom(this, null, null, new Tester() + { + public boolean isDone() + { + return false; + } + }); + iterator = partition.unfilteredIterator(columns, Slices.ALL, true); + } + return iterator.hasNext(); + } + + public Unfiltered next() throws IOException + { + if (!hasNext()) + throw new NoSuchElementException(); + return iterator.next(); + } + + public Iterator slice(final Slice slice) throws IOException + { + if (partition == null) + { + partition = createBuffer(1); + partition.populateFrom(this, slice.start(), slice.end(), new Tester() + { + public boolean isDone() + { + return false; + } + }); + } + + return partition.unfilteredIterator(columns, Slices.with(metadata().comparator, slice), true); + } + } + + private class ReverseIndexedReader extends IndexedReader + { + private ReusablePartitionData partition; + private UnfilteredRowIterator iterator; + + private ReverseIndexedReader(RowIndexEntry indexEntry, FileDataInput file, boolean isAtPartitionStart, boolean shouldCloseFile) + { + super(file, shouldCloseFile, indexEntry, isAtPartitionStart); + this.currentIndexIdx = indexEntry.columnsIndex().size(); + } + + public boolean hasNext() throws IOException + { + // If it's called before we've created the file, create it. This then mean + // we're reading from the end of the partition. + if (!isInit) + { + seekToPosition(indexEntry.position); + ByteBufferUtil.skipShortLength(file); // partition key + DeletionTime.serializer.skip(file); // partition deletion + if (sstable.header.hasStatic()) + UnfilteredSerializer.serializer.skipStaticRow(file, sstable.header, helper); + isInit = true; + } + + if (partition == null) + { + partition = createBuffer(indexes.size()); + partition.populateFrom(this, null, null, new Tester() + { + public boolean isDone() + { + return false; + } + }); + iterator = partition.unfilteredIterator(columns, Slices.ALL, true); + } + + return iterator.hasNext(); + } + + public Unfiltered next() throws IOException + { + if (!hasNext()) + throw new NoSuchElementException(); + return iterator.next(); + } + + private void prepareBlock(int blockIdx, Slice.Bound start, Slice.Bound end) throws IOException + { + updateBlock(blockIdx); + + if (partition == null) + partition = createBuffer(indexes.size()); + else + partition.clear(); + + final FileMark fileMark = mark; + final long width = currentIndex().width; + + partition.populateFrom(this, start, end, new Tester() + { + public boolean isDone() + { + return file.bytesPastMark(fileMark) >= width; + } + }); + } + + @Override + public Iterator slice(final Slice slice) throws IOException + { + // if our previous slicing already got us the smallest row in the sstable, we're done + if (currentIndexIdx < 0) + return Collections.emptyIterator(); + + final List indexes = indexEntry.columnsIndex(); + + // Find the first index block we'll need to read for the slice. + final int startIdx = IndexHelper.indexFor(slice.end(), indexes, sstable.metadata.comparator, true, currentIndexIdx); + if (startIdx < 0) + return Collections.emptyIterator(); + + // Find the last index block we'll need to read for the slice. + int lastIdx = IndexHelper.indexFor(slice.start(), indexes, sstable.metadata.comparator, true, startIdx); + + // The index search is by firstname and so lastIdx is such that + // indexes[lastIdx].firstName < slice.start <= indexes[lastIdx + 1].firstName + // However, if indexes[lastIdx].lastName < slice.start we can bump lastIdx. + if (lastIdx >= 0 && metadata().comparator.compare(indexes.get(lastIdx).lastName, slice.start()) < 0) + ++lastIdx; + + final int endIdx = lastIdx; + + // Because we're reversed, even if it is our current block, we should re-prepare the block since we would + // have skipped anything not in the previous slice. + prepareBlock(startIdx, slice.start(), slice.end()); + + return new AbstractIterator() + { + private Iterator currentBlockIterator = partition.unfilteredIterator(columns, Slices.with(metadata().comparator, slice), true); + + protected Unfiltered computeNext() + { + try + { + if (currentBlockIterator.hasNext()) + return currentBlockIterator.next(); + + --currentIndexIdx; + if (currentIndexIdx < 0 || currentIndexIdx < endIdx) + return endOfData(); + + // Note that since we know we're read blocks backward, there is no point in checking the slice end, so we pass null + prepareBlock(currentIndexIdx, slice.start(), null); + currentBlockIterator = partition.unfilteredIterator(columns, Slices.with(metadata().comparator, slice), true); + return computeNext(); + } + catch (IOException e) + { + try + { + close(); + } + catch (IOException suppressed) + { + e.addSuppressed(suppressed); + } + sstable.markSuspect(); + throw new CorruptSSTableException(e, file.getPath()); + } + } + }; + } + } + + private abstract class Tester + { + public abstract boolean isDone(); + } + + private class ReusablePartitionData extends AbstractPartitionData + { + private final Writer rowWriter; + private final RangeTombstoneCollector markerWriter; + + private ReusablePartitionData(CFMetaData metadata, + DecoratedKey partitionKey, + DeletionTime deletionTime, + PartitionColumns columns, + int initialRowCapacity) + { + super(metadata, partitionKey, deletionTime, columns, initialRowCapacity, false); + + this.rowWriter = new Writer(true); + // Note that even though the iterator handles the reverse case, this object holds the data for a single index bock, and we read index blocks in + // forward clustering order. + this.markerWriter = new RangeTombstoneCollector(false); + } + + // Note that this method is here rather than in the readers because we want to use it for both readers and they + // don't extend one another + private void populateFrom(Reader reader, Slice.Bound start, Slice.Bound end, Tester tester) throws IOException + { + // If we have a start bound, skip everything that comes before it. + while (reader.deserializer.hasNext() && start != null && reader.deserializer.compareNextTo(start) <= 0 && !tester.isDone()) + { + if (reader.deserializer.nextIsRow()) + reader.deserializer.skipNext(); + else + reader.updateOpenMarker((RangeTombstoneMarker)reader.deserializer.readNext()); + } + + // If we have an open marker, it's either one from what we just skipped (if start != null), or it's from the previous index block. + if (reader.openMarker != null) + { + // If we have no start but still an openMarker, this means we're indexed and it's coming from the previous block + Slice.Bound markerStart = start; + if (start == null) + { + ClusteringPrefix c = ((IndexedReader)reader).previousIndex().lastName; + markerStart = Slice.Bound.exclusiveStartOf(c); + } + writeMarker(markerStart, reader.openMarker); + } + + // Now deserialize everything until we reach our requested end (if we have one) + while (reader.deserializer.hasNext() + && (end == null || reader.deserializer.compareNextTo(end) <= 0) + && !tester.isDone()) + { + Unfiltered unfiltered = reader.deserializer.readNext(); + if (unfiltered.kind() == Unfiltered.Kind.ROW) + { + ((Row) unfiltered).copyTo(rowWriter); + } + else + { + RangeTombstoneMarker marker = (RangeTombstoneMarker) unfiltered; + reader.updateOpenMarker(marker); + marker.copyTo(markerWriter); + } + } + + // If we have an open marker, we should close it before finishing + if (reader.openMarker != null) + { + // If we no end and still an openMarker, this means we're indexed and the marker can be close using the blocks end + Slice.Bound markerEnd = end; + if (end == null) + { + ClusteringPrefix c = ((IndexedReader)reader).currentIndex().lastName; + markerEnd = Slice.Bound.inclusiveEndOf(c); + } + writeMarker(markerEnd, reader.getAndClearOpenMarker()); + } + } + + private void writeMarker(Slice.Bound bound, DeletionTime dt) + { + bound.writeTo(markerWriter); + markerWriter.writeBoundDeletion(dt); + markerWriter.endOfMarker(); + } + + @Override + public void clear() + { + super.clear(); + rowWriter.reset(); + markerWriter.reset(); + } + } +} diff --git a/src/java/org/apache/cassandra/db/commitlog/CommitLogArchiver.java b/src/java/org/apache/cassandra/db/commitlog/CommitLogArchiver.java index 02072de6ab..ec270ddb6c 100644 --- a/src/java/org/apache/cassandra/db/commitlog/CommitLogArchiver.java +++ b/src/java/org/apache/cassandra/db/commitlog/CommitLogArchiver.java @@ -206,7 +206,7 @@ public class CommitLogArchiver descriptor = fromHeader; else descriptor = fromName; - if (descriptor.version > CommitLogDescriptor.VERSION_22) + if (descriptor.version > CommitLogDescriptor.current_version) throw new IllegalStateException("Unsupported commit log version: " + descriptor.version); if (descriptor.compression != null) { diff --git a/src/java/org/apache/cassandra/db/commitlog/CommitLogDescriptor.java b/src/java/org/apache/cassandra/db/commitlog/CommitLogDescriptor.java index c4728fdafc..1eb640eee7 100644 --- a/src/java/org/apache/cassandra/db/commitlog/CommitLogDescriptor.java +++ b/src/java/org/apache/cassandra/db/commitlog/CommitLogDescriptor.java @@ -57,12 +57,13 @@ public class CommitLogDescriptor public static final int VERSION_20 = 3; public static final int VERSION_21 = 4; public static final int VERSION_22 = 5; + public static final int VERSION_30 = 6; /** * Increment this number if there is a changes in the commit log disc layout or MessagingVersion changes. * Note: make sure to handle {@link #getMessagingVersion()} */ @VisibleForTesting - public static final int current_version = VERSION_22; + public static final int current_version = VERSION_30; final int version; public final long id; @@ -195,6 +196,8 @@ public class CommitLogDescriptor return MessagingService.VERSION_21; case VERSION_22: return MessagingService.VERSION_22; + case VERSION_30: + return MessagingService.VERSION_30; default: throw new IllegalStateException("Unknown commitlog version " + version); } diff --git a/src/java/org/apache/cassandra/db/commitlog/CommitLogReplayer.java b/src/java/org/apache/cassandra/db/commitlog/CommitLogReplayer.java index 176f64bb2a..902f1c4028 100644 --- a/src/java/org/apache/cassandra/db/commitlog/CommitLogReplayer.java +++ b/src/java/org/apache/cassandra/db/commitlog/CommitLogReplayer.java @@ -47,6 +47,8 @@ import org.apache.cassandra.concurrent.StageManager; import org.apache.cassandra.config.CFMetaData; import org.apache.cassandra.config.Schema; import org.apache.cassandra.db.*; +import org.apache.cassandra.db.rows.SerializationHelper; +import org.apache.cassandra.db.partitions.PartitionUpdate; import org.apache.cassandra.exceptions.ConfigurationException; import org.apache.cassandra.io.compress.CompressionParameters; import org.apache.cassandra.io.compress.ICompressor; @@ -55,7 +57,6 @@ import org.apache.cassandra.io.util.FastByteArrayInputStream; import org.apache.cassandra.io.util.FileDataInput; import org.apache.cassandra.io.util.FileUtils; import org.apache.cassandra.io.util.RandomAccessReader; -import org.apache.cassandra.utils.ByteBufferUtil; import org.apache.cassandra.utils.CRC32Factory; import org.apache.cassandra.utils.FBUtilities; import org.apache.cassandra.utils.JVMStabilityInspector; @@ -195,7 +196,7 @@ public class CommitLogReplayer abstract static class ReplayFilter { - public abstract Iterable filter(Mutation mutation); + public abstract Iterable filter(Mutation mutation); public abstract boolean includes(CFMetaData metadata); @@ -227,9 +228,9 @@ public class CommitLogReplayer private static class AlwaysReplayFilter extends ReplayFilter { - public Iterable filter(Mutation mutation) + public Iterable filter(Mutation mutation) { - return mutation.getColumnFamilies(); + return mutation.getPartitionUpdates(); } public boolean includes(CFMetaData metadata) @@ -247,17 +248,17 @@ public class CommitLogReplayer this.toReplay = toReplay; } - public Iterable filter(Mutation mutation) + public Iterable filter(Mutation mutation) { final Collection cfNames = toReplay.get(mutation.getKeyspaceName()); if (cfNames == null) return Collections.emptySet(); - return Iterables.filter(mutation.getColumnFamilies(), new Predicate() + return Iterables.filter(mutation.getPartitionUpdates(), new Predicate() { - public boolean apply(ColumnFamily cf) + public boolean apply(PartitionUpdate upd) { - return cfNames.contains(cf.metadata().cfName); + return cfNames.contains(upd.metadata().cfName); } }); } @@ -330,7 +331,8 @@ public class CommitLogReplayer { int uncompressedLength = reader.readInt(); replayEnd = replayPos + uncompressedLength; - } else + } + else { replayEnd = end; } @@ -478,11 +480,10 @@ public class CommitLogReplayer { mutation = Mutation.serializer.deserialize(new DataInputStream(bufIn), desc.getMessagingVersion(), - ColumnSerializer.Flag.LOCAL); + SerializationHelper.Flag.LOCAL); // doublecheck that what we read is [still] valid for the current schema - for (ColumnFamily cf : mutation.getColumnFamilies()) - for (Cell cell : cf) - cf.getComparator().validate(cell.name()); + for (PartitionUpdate upd : mutation.getPartitionUpdates()) + upd.validate(); } catch (UnknownColumnFamilyException ex) { @@ -515,7 +516,7 @@ public class CommitLogReplayer } if (logger.isDebugEnabled()) - logger.debug("replaying mutation for {}.{}: {}", mutation.getKeyspaceName(), ByteBufferUtil.bytesToHex(mutation.key()), "{" + StringUtils.join(mutation.getColumnFamilies().iterator(), ", ") + "}"); + logger.debug("replaying mutation for {}.{}: {}", mutation.getKeyspaceName(), mutation.key(), "{" + StringUtils.join(mutation.getPartitionUpdates().iterator(), ", ") + "}"); Runnable runnable = new WrappedRunnable() { @@ -534,12 +535,12 @@ public class CommitLogReplayer // or c) are part of a cf that was dropped. // Keep in mind that the cf.name() is suspect. do every thing based on the cfid instead. Mutation newMutation = null; - for (ColumnFamily columnFamily : replayFilter.filter(mutation)) + for (PartitionUpdate update : replayFilter.filter(mutation)) { - if (Schema.instance.getCF(columnFamily.id()) == null) + if (Schema.instance.getCF(update.metadata().cfId) == null) continue; // dropped - ReplayPosition rp = cfPositions.get(columnFamily.id()); + ReplayPosition rp = cfPositions.get(update.metadata().cfId); // replay if current segment is newer than last flushed one or, // if it is the last known segment, if we are after the replay position @@ -547,7 +548,7 @@ public class CommitLogReplayer { if (newMutation == null) newMutation = new Mutation(mutation.getKeyspaceName(), mutation.key()); - newMutation.add(columnFamily); + newMutation.add(update); replayedCount.incrementAndGet(); } } @@ -571,9 +572,9 @@ public class CommitLogReplayer { long restoreTarget = CommitLog.instance.archiver.restorePointInTime; - for (ColumnFamily families : fm.getColumnFamilies()) + for (PartitionUpdate upd : fm.getPartitionUpdates()) { - if (CommitLog.instance.archiver.precision.toMillis(families.maxTimestamp()) > restoreTarget) + if (CommitLog.instance.archiver.precision.toMillis(upd.maxTimestamp()) > restoreTarget) return true; } return false; diff --git a/src/java/org/apache/cassandra/db/commitlog/CommitLogSegment.java b/src/java/org/apache/cassandra/db/commitlog/CommitLogSegment.java index d748006794..7473845aa0 100644 --- a/src/java/org/apache/cassandra/db/commitlog/CommitLogSegment.java +++ b/src/java/org/apache/cassandra/db/commitlog/CommitLogSegment.java @@ -45,8 +45,8 @@ import org.slf4j.LoggerFactory; import org.apache.cassandra.config.CFMetaData; import org.apache.cassandra.config.DatabaseDescriptor; import org.apache.cassandra.config.Schema; -import org.apache.cassandra.db.ColumnFamily; import org.apache.cassandra.db.Mutation; +import org.apache.cassandra.db.partitions.PartitionUpdate; import org.apache.cassandra.io.FSWriteError; import org.apache.cassandra.io.util.FileUtils; import org.apache.cassandra.utils.CLibrary; @@ -398,12 +398,12 @@ public abstract class CommitLogSegment void markDirty(Mutation mutation, int allocatedPosition) { - for (ColumnFamily columnFamily : mutation.getColumnFamilies()) + for (PartitionUpdate update : mutation.getPartitionUpdates()) { // check for deleted CFS - CFMetaData cfm = columnFamily.metadata(); + CFMetaData cfm = update.metadata(); if (cfm.isPurged()) - logger.error("Attempted to write commit log entry for unrecognized table: {}", columnFamily.id()); + logger.error("Attempted to write commit log entry for unrecognized table: {}", cfm.cfId); else ensureAtleast(cfDirty, cfm.cfId, allocatedPosition); } diff --git a/src/java/org/apache/cassandra/db/compaction/AbstractCompactedRow.java b/src/java/org/apache/cassandra/db/compaction/AbstractCompactedRow.java deleted file mode 100644 index 16b5fac827..0000000000 --- a/src/java/org/apache/cassandra/db/compaction/AbstractCompactedRow.java +++ /dev/null @@ -1,65 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.apache.cassandra.db.compaction; - -import java.io.Closeable; -import java.io.IOException; -import java.security.MessageDigest; - -import org.apache.cassandra.db.DecoratedKey; -import org.apache.cassandra.db.RowIndexEntry; -import org.apache.cassandra.io.sstable.ColumnStats; -import org.apache.cassandra.io.util.SequentialWriter; - -/** - * a CompactedRow is an object that takes a bunch of rows (keys + columnfamilies) - * and can write a compacted version of those rows to an output stream. It does - * NOT necessarily require creating a merged CF object in memory. - */ -public abstract class AbstractCompactedRow implements Closeable -{ - public final DecoratedKey key; - - public AbstractCompactedRow(DecoratedKey key) - { - this.key = key; - } - - /** - * write the row (size + column index + filter + column data, but NOT row key) to @param out. - * - * write() may change internal state; it is NOT valid to call write() or update() a second time. - * - * @return index information for the written row, or null if the compaction resulted in only expired tombstones. - */ - public abstract RowIndexEntry write(long currentPosition, SequentialWriter out) throws IOException; - - /** - * update @param digest with the data bytes of the row (not including row key or row size). - * May be called even if empty. - * - * update() may change internal state; it is NOT valid to call write() or update() a second time. - */ - public abstract void update(MessageDigest digest); - - /** - * @return aggregate information about the columns in this row. Some fields may - * contain default values if computing them value would require extra effort we're not willing to make. - */ - public abstract ColumnStats columnStats(); -} diff --git a/src/java/org/apache/cassandra/db/compaction/AbstractCompactionIterable.java b/src/java/org/apache/cassandra/db/compaction/AbstractCompactionIterable.java deleted file mode 100644 index 9fe8fd9aa2..0000000000 --- a/src/java/org/apache/cassandra/db/compaction/AbstractCompactionIterable.java +++ /dev/null @@ -1,83 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.apache.cassandra.db.compaction; - -import java.util.List; -import java.util.UUID; -import java.util.concurrent.atomic.AtomicLong; - -import org.apache.cassandra.io.sstable.ISSTableScanner; -import org.apache.cassandra.utils.CloseableIterator; - -public abstract class AbstractCompactionIterable extends CompactionInfo.Holder implements Iterable -{ - protected final OperationType type; - protected final CompactionController controller; - protected final long totalBytes; - protected volatile long bytesRead; - protected final List scanners; - protected final UUID compactionId; - /* - * counters for merged rows. - * array index represents (number of merged rows - 1), so index 0 is counter for no merge (1 row), - * index 1 is counter for 2 rows merged, and so on. - */ - protected final AtomicLong[] mergeCounters; - - public AbstractCompactionIterable(CompactionController controller, OperationType type, List scanners, UUID compactionId) - { - this.controller = controller; - this.type = type; - this.scanners = scanners; - this.bytesRead = 0; - this.compactionId = compactionId; - - long bytes = 0; - for (ISSTableScanner scanner : scanners) - bytes += scanner.getLengthInBytes(); - this.totalBytes = bytes; - mergeCounters = new AtomicLong[scanners.size()]; - for (int i = 0; i < mergeCounters.length; i++) - mergeCounters[i] = new AtomicLong(); - } - - public CompactionInfo getCompactionInfo() - { - return new CompactionInfo(controller.cfs.metadata, - type, - bytesRead, - totalBytes, - compactionId); - } - - protected void updateCounterFor(int rows) - { - assert rows > 0 && rows - 1 < mergeCounters.length; - mergeCounters[rows - 1].incrementAndGet(); - } - - public long[] getMergedRowCounts() - { - long[] counters = new long[mergeCounters.length]; - for (int i = 0; i < counters.length; i++) - counters[i] = mergeCounters[i].get(); - return counters; - } - - public abstract CloseableIterator iterator(); -} diff --git a/src/java/org/apache/cassandra/db/compaction/AbstractCompactionStrategy.java b/src/java/org/apache/cassandra/db/compaction/AbstractCompactionStrategy.java index 28144ca1cf..d8499ea93b 100644 --- a/src/java/org/apache/cassandra/db/compaction/AbstractCompactionStrategy.java +++ b/src/java/org/apache/cassandra/db/compaction/AbstractCompactionStrategy.java @@ -36,6 +36,7 @@ import org.apache.cassandra.dht.Token; import org.apache.cassandra.exceptions.ConfigurationException; import org.apache.cassandra.io.sstable.Component; import org.apache.cassandra.io.sstable.ISSTableScanner; +import org.apache.cassandra.utils.FBUtilities; import org.apache.cassandra.utils.JVMStabilityInspector; /** diff --git a/src/java/org/apache/cassandra/db/compaction/CompactionController.java b/src/java/org/apache/cassandra/db/compaction/CompactionController.java index 81d8b7cff2..303de152d6 100644 --- a/src/java/org/apache/cassandra/db/compaction/CompactionController.java +++ b/src/java/org/apache/cassandra/db/compaction/CompactionController.java @@ -27,7 +27,7 @@ import org.apache.cassandra.db.ColumnFamilyStore; import org.apache.cassandra.db.lifecycle.SSTableIntervalTree; import org.apache.cassandra.db.lifecycle.Tracker; import org.apache.cassandra.db.DecoratedKey; -import org.apache.cassandra.db.RowPosition; +import org.apache.cassandra.db.PartitionPosition; import org.apache.cassandra.utils.AlwaysPresentFilter; import org.apache.cassandra.utils.OverlapIterator; @@ -44,7 +44,7 @@ public class CompactionController implements AutoCloseable public final ColumnFamilyStore cfs; private Refs overlappingSSTables; - private OverlapIterator overlapIterator; + private OverlapIterator overlapIterator; private final Iterable compacting; public final int gcBefore; @@ -189,9 +189,9 @@ public class CompactionController implements AutoCloseable return min; } - public void invalidateCachedRow(DecoratedKey key) + public void invalidateCachedPartition(DecoratedKey key) { - cfs.invalidateCachedRow(key); + cfs.invalidateCachedPartition(key); } public void close() diff --git a/src/java/org/apache/cassandra/db/compaction/CompactionIterable.java b/src/java/org/apache/cassandra/db/compaction/CompactionIterable.java deleted file mode 100644 index 23d8a4a47d..0000000000 --- a/src/java/org/apache/cassandra/db/compaction/CompactionIterable.java +++ /dev/null @@ -1,96 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.apache.cassandra.db.compaction; - -import java.util.ArrayList; -import java.util.Comparator; -import java.util.List; -import java.util.UUID; - -import com.google.common.collect.ImmutableList; - -import org.apache.cassandra.db.columniterator.OnDiskAtomIterator; -import org.apache.cassandra.io.sstable.format.SSTableFormat; -import org.apache.cassandra.io.sstable.ISSTableScanner; -import org.apache.cassandra.utils.CloseableIterator; -import org.apache.cassandra.utils.MergeIterator; - -public class CompactionIterable extends AbstractCompactionIterable -{ - final SSTableFormat format; - - private static final Comparator comparator = new Comparator() - { - public int compare(OnDiskAtomIterator i1, OnDiskAtomIterator i2) - { - return i1.getKey().compareTo(i2.getKey()); - } - }; - - public CompactionIterable(OperationType type, - List scanners, - CompactionController controller, - SSTableFormat.Type formatType, - UUID compactionId) - { - super(controller, type, scanners, compactionId); - this.format = formatType.info; - } - - public CloseableIterator iterator() - { - return MergeIterator.get(scanners, comparator, new Reducer()); - } - - public String toString() - { - return this.getCompactionInfo().toString(); - } - - protected class Reducer extends MergeIterator.Reducer - { - protected final List rows = new ArrayList<>(); - - public void reduce(OnDiskAtomIterator current) - { - rows.add(current); - } - - protected AbstractCompactedRow getReduced() - { - assert !rows.isEmpty(); - - CompactionIterable.this.updateCounterFor(rows.size()); - try - { - // create a new container for rows, since we're going to clear ours for the next one, - // and the AbstractCompactionRow code should be able to assume that the collection it receives - // won't be pulled out from under it. - return format.getCompactedRowWriter(controller, ImmutableList.copyOf(rows)); - } - finally - { - rows.clear(); - long n = 0; - for (ISSTableScanner scanner : scanners) - n += scanner.getCurrentPosition(); - bytesRead = n; - } - } - } -} diff --git a/src/java/org/apache/cassandra/db/compaction/CompactionIterator.java b/src/java/org/apache/cassandra/db/compaction/CompactionIterator.java new file mode 100644 index 0000000000..b3cb370fb3 --- /dev/null +++ b/src/java/org/apache/cassandra/db/compaction/CompactionIterator.java @@ -0,0 +1,299 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.cassandra.db.compaction; + +import java.util.UUID; +import java.util.List; + +import org.apache.cassandra.config.ColumnDefinition; +import org.apache.cassandra.db.*; +import org.apache.cassandra.db.index.SecondaryIndexManager; +import org.apache.cassandra.db.partitions.*; +import org.apache.cassandra.db.rows.*; +import org.apache.cassandra.io.sstable.ISSTableScanner; +import org.apache.cassandra.io.sstable.format.SSTableFormat; +import org.apache.cassandra.metrics.CompactionMetrics; + +/** + * Merge multiple iterators over the content of sstable into a "compacted" iterator. + *

+ * On top of the actual merging the source iterators, this class: + *

    + *
  • purge gc-able tombstones if possible (see PurgingPartitionIterator below).
    • + *
  • update 2ndary indexes if necessary (as we don't read-before-write on index updates, index entries are + * not deleted on deletion of the base table data, which is ok because we'll fix index inconsistency + * on reads. This however mean that potentially obsolete index entries could be kept a long time for + * data that is not read often, so compaction "pro-actively" fix such index entries. This is mainly + * an optimization).
    • + *
  • invalidate cached partitions that are empty post-compaction. This avoids keeping partitions with + * only purgable tombstones in the row cache.
    • + *
  • keep tracks of the compaction progress.
    • + *
+ */ +public class CompactionIterator extends CompactionInfo.Holder implements UnfilteredPartitionIterator +{ + private static final long UNFILTERED_TO_UPDATE_PROGRESS = 100; + + private final OperationType type; + private final CompactionController controller; + private final List scanners; + private final int nowInSec; + private final UUID compactionId; + + private final long totalBytes; + private long bytesRead; + + /* + * counters for merged rows. + * array index represents (number of merged rows - 1), so index 0 is counter for no merge (1 row), + * index 1 is counter for 2 rows merged, and so on. + */ + private final long[] mergeCounters; + + private final UnfilteredPartitionIterator mergedIterator; + private final CompactionMetrics metrics; + + // The number of row/RT merged by the iterator + private int merged; + + public CompactionIterator(OperationType type, List scanners, CompactionController controller, int nowInSec, UUID compactionId) + { + this(type, scanners, controller, nowInSec, compactionId, null); + } + + @SuppressWarnings("resource") // We make sure to close mergedIterator in close() and CompactionIterator is itself an AutoCloseable + public CompactionIterator(OperationType type, List scanners, CompactionController controller, int nowInSec, UUID compactionId, CompactionMetrics metrics) + { + this.controller = controller; + this.type = type; + this.scanners = scanners; + this.nowInSec = nowInSec; + this.compactionId = compactionId; + this.bytesRead = 0; + + long bytes = 0; + for (ISSTableScanner scanner : scanners) + bytes += scanner.getLengthInBytes(); + this.totalBytes = bytes; + this.mergeCounters = new long[scanners.size()]; + this.metrics = metrics; + + if (metrics != null) + metrics.beginCompaction(this); + + this.mergedIterator = scanners.isEmpty() + ? UnfilteredPartitionIterators.EMPTY + : UnfilteredPartitionIterators.convertExpiredCellsToTombstones(new PurgingPartitionIterator(UnfilteredPartitionIterators.merge(scanners, nowInSec, listener()), controller), nowInSec); + } + + public boolean isForThrift() + { + return false; + } + + public CompactionInfo getCompactionInfo() + { + return new CompactionInfo(controller.cfs.metadata, + type, + bytesRead, + totalBytes, + compactionId); + } + + private void updateCounterFor(int rows) + { + assert rows > 0 && rows - 1 < mergeCounters.length; + mergeCounters[rows - 1] += 1; + } + + public long[] getMergedRowCounts() + { + return mergeCounters; + } + + private UnfilteredPartitionIterators.MergeListener listener() + { + return new UnfilteredPartitionIterators.MergeListener() + { + public UnfilteredRowIterators.MergeListener getRowMergeListener(DecoratedKey partitionKey, List versions) + { + int merged = 0; + for (UnfilteredRowIterator iter : versions) + { + if (iter != null) + merged++; + } + + assert merged > 0; + + CompactionIterator.this.updateCounterFor(merged); + + /* + * The row level listener does 2 things: + * - It updates 2ndary indexes for deleted/shadowed cells + * - It updates progress regularly (every UNFILTERED_TO_UPDATE_PROGRESS) + */ + final SecondaryIndexManager.Updater indexer = type == OperationType.COMPACTION + ? controller.cfs.indexManager.gcUpdaterFor(partitionKey, nowInSec) + : SecondaryIndexManager.nullUpdater; + + return new UnfilteredRowIterators.MergeListener() + { + private Clustering clustering; + + public void onMergePartitionLevelDeletion(DeletionTime mergedDeletion, DeletionTime[] versions) + { + } + + public void onMergingRows(Clustering clustering, LivenessInfo mergedInfo, DeletionTime mergedDeletion, Row[] versions) + { + this.clustering = clustering; + } + + public void onMergedComplexDeletion(ColumnDefinition c, DeletionTime mergedCompositeDeletion, DeletionTime[] versions) + { + } + + public void onMergedCells(Cell mergedCell, Cell[] versions) + { + if (indexer == SecondaryIndexManager.nullUpdater) + return; + + for (int i = 0; i < versions.length; i++) + { + Cell version = versions[i]; + if (version != null && (mergedCell == null || !mergedCell.equals(version))) + indexer.remove(clustering, version); + } + } + + public void onRowDone() + { + int merged = ++CompactionIterator.this.merged; + if (merged % UNFILTERED_TO_UPDATE_PROGRESS == 0) + updateBytesRead(); + } + + public void onMergedRangeTombstoneMarkers(RangeTombstoneMarker mergedMarker, RangeTombstoneMarker[] versions) + { + int merged = ++CompactionIterator.this.merged; + if (merged % UNFILTERED_TO_UPDATE_PROGRESS == 0) + updateBytesRead(); + } + + public void close() + { + } + }; + } + + public void close() + { + } + }; + } + + private void updateBytesRead() + { + long n = 0; + for (ISSTableScanner scanner : scanners) + n += scanner.getCurrentPosition(); + bytesRead = n; + } + + public boolean hasNext() + { + return mergedIterator.hasNext(); + } + + public UnfilteredRowIterator next() + { + return mergedIterator.next(); + } + + public void remove() + { + throw new UnsupportedOperationException(); + } + + public void close() + { + try + { + mergedIterator.close(); + } + finally + { + if (metrics != null) + metrics.finishCompaction(this); + } + } + + public String toString() + { + return this.getCompactionInfo().toString(); + } + + private class PurgingPartitionIterator extends TombstonePurgingPartitionIterator + { + private final CompactionController controller; + + private DecoratedKey currentKey; + private long maxPurgeableTimestamp; + private boolean hasCalculatedMaxPurgeableTimestamp; + + private PurgingPartitionIterator(UnfilteredPartitionIterator toPurge, CompactionController controller) + { + super(toPurge, controller.gcBefore); + this.controller = controller; + } + + @Override + protected void onEmpty(DecoratedKey key) + { + if (type == OperationType.COMPACTION) + controller.cfs.invalidateCachedPartition(key); + } + + @Override + protected boolean shouldFilter(UnfilteredRowIterator iterator) + { + currentKey = iterator.partitionKey(); + hasCalculatedMaxPurgeableTimestamp = false; + + // TODO: we could be able to skip filtering if UnfilteredRowIterator was giving us some stats + // (like the smallest local deletion time). + return true; + } + + /* + * Tombstones with a localDeletionTime before this can be purged. This is the minimum timestamp for any sstable + * containing `currentKey` outside of the set of sstables involved in this compaction. This is computed lazily + * on demand as we only need this if there is tombstones and this a bit expensive (see #8914). + */ + protected long getMaxPurgeableTimestamp() + { + if (!hasCalculatedMaxPurgeableTimestamp) + { + hasCalculatedMaxPurgeableTimestamp = true; + maxPurgeableTimestamp = controller.maxPurgeableTimestamp(currentKey); + } + return maxPurgeableTimestamp; + } + } +} diff --git a/src/java/org/apache/cassandra/db/compaction/CompactionManager.java b/src/java/org/apache/cassandra/db/compaction/CompactionManager.java index 763871e463..a6c3d8c6d7 100644 --- a/src/java/org/apache/cassandra/db/compaction/CompactionManager.java +++ b/src/java/org/apache/cassandra/db/compaction/CompactionManager.java @@ -41,7 +41,6 @@ import javax.management.ObjectName; import javax.management.openmbean.OpenDataException; import javax.management.openmbean.TabularData; -import com.google.common.base.Predicate; import com.google.common.base.Throwables; import com.google.common.collect.*; import com.google.common.util.concurrent.*; @@ -57,6 +56,7 @@ import org.apache.cassandra.config.DatabaseDescriptor; import org.apache.cassandra.config.Schema; import org.apache.cassandra.db.*; import org.apache.cassandra.db.compaction.CompactionInfo.Holder; +import org.apache.cassandra.db.rows.*; import org.apache.cassandra.db.index.SecondaryIndexBuilder; import org.apache.cassandra.db.lifecycle.LifecycleTransaction; import org.apache.cassandra.dht.Bounds; @@ -71,7 +71,6 @@ import org.apache.cassandra.metrics.CompactionMetrics; import org.apache.cassandra.repair.Validator; import org.apache.cassandra.service.ActiveRepairService; import org.apache.cassandra.service.StorageService; -import org.apache.cassandra.utils.CloseableIterator; import org.apache.cassandra.utils.FBUtilities; import org.apache.cassandra.utils.JVMStabilityInspector; import org.apache.cassandra.utils.MerkleTree; @@ -235,7 +234,7 @@ public class CompactionManager implements CompactionManagerMBean } CompactionStrategyManager strategy = cfs.getCompactionStrategyManager(); - AbstractCompactionTask task = strategy.getNextBackgroundTask(getDefaultGcBefore(cfs)); + AbstractCompactionTask task = strategy.getNextBackgroundTask(getDefaultGcBefore(cfs, FBUtilities.nowInSeconds())); if (task == null) { logger.debug("No tasks available"); @@ -426,7 +425,7 @@ public class CompactionManager implements CompactionManagerMBean @Override public void execute(LifecycleTransaction txn) throws IOException { - CleanupStrategy cleanupStrategy = CleanupStrategy.get(cfStore, ranges); + CleanupStrategy cleanupStrategy = CleanupStrategy.get(cfStore, ranges, FBUtilities.nowInSeconds()); doCleanupOne(cfStore, txn, cleanupStrategy, ranges, hasIndexes); } }, OperationType.CLEANUP); @@ -541,7 +540,7 @@ public class CompactionManager implements CompactionManagerMBean public void performMaximal(final ColumnFamilyStore cfStore, boolean splitOutput) { - FBUtilities.waitOnFutures(submitMaximal(cfStore, getDefaultGcBefore(cfStore), splitOutput)); + FBUtilities.waitOnFutures(submitMaximal(cfStore, getDefaultGcBefore(cfStore, FBUtilities.nowInSeconds()), splitOutput)); } public List> submitMaximal(final ColumnFamilyStore cfStore, final int gcBefore, boolean splitOutput) @@ -595,8 +594,9 @@ public class CompactionManager implements CompactionManagerMBean } List> futures = new ArrayList<>(); + int nowInSec = FBUtilities.nowInSeconds(); for (ColumnFamilyStore cfs : descriptors.keySet()) - futures.add(submitUserDefined(cfs, descriptors.get(cfs), getDefaultGcBefore(cfs))); + futures.add(submitUserDefined(cfs, descriptors.get(cfs), getDefaultGcBefore(cfs, nowInSec))); FBUtilities.waitOnFutures(futures); } @@ -817,29 +817,29 @@ public class CompactionManager implements CompactionManagerMBean if (compactionFileLocation == null) throw new IOException("disk full"); - ISSTableScanner scanner = cleanupStrategy.getScanner(sstable, getRateLimiter()); - CleanupInfo ci = new CleanupInfo(sstable, scanner); - - metrics.beginCompaction(ci); List finished; + int nowInSec = FBUtilities.nowInSeconds(); try (SSTableRewriter writer = new SSTableRewriter(cfs, txn, sstable.maxDataAge, false); - CompactionController controller = new CompactionController(cfs, txn.originals(), getDefaultGcBefore(cfs))) + ISSTableScanner scanner = cleanupStrategy.getScanner(sstable, getRateLimiter()); + CompactionController controller = new CompactionController(cfs, txn.originals(), getDefaultGcBefore(cfs, nowInSec)); + CompactionIterator ci = new CompactionIterator(OperationType.CLEANUP, Collections.singletonList(scanner), controller, nowInSec, UUIDGen.getTimeUUID(), metrics)) { writer.switchWriter(createWriter(cfs, compactionFileLocation, expectedBloomFilterSize, sstable.getSSTableMetadata().repairedAt, sstable)); - while (scanner.hasNext()) + while (ci.hasNext()) { if (ci.isStopRequested()) throw new CompactionInterruptedException(ci.getCompactionInfo()); - @SuppressWarnings("resource") - SSTableIdentityIterator row = cleanupStrategy.cleanup((SSTableIdentityIterator) scanner.next()); - if (row == null) - continue; - @SuppressWarnings("resource") - AbstractCompactedRow compactedRow = new LazilyCompactedRow(controller, Collections.singletonList(row)); - if (writer.append(compactedRow) != null) - totalkeysWritten++; + try (UnfilteredRowIterator partition = ci.next(); + UnfilteredRowIterator notCleaned = cleanupStrategy.cleanup(partition)) + { + if (notCleaned == null) + continue; + + if (writer.append(notCleaned) != null) + totalkeysWritten++; + } } // flush to ensure we don't lose the tombstones on a restart, since they are not commitlog'd @@ -847,11 +847,6 @@ public class CompactionManager implements CompactionManagerMBean finished = writer.finish(); } - finally - { - scanner.close(); - metrics.finishCompaction(ci); - } if (!finished.isEmpty()) { @@ -869,23 +864,30 @@ public class CompactionManager implements CompactionManagerMBean private static abstract class CleanupStrategy { - public static CleanupStrategy get(ColumnFamilyStore cfs, Collection> ranges) + protected final Collection> ranges; + protected final int nowInSec; + + protected CleanupStrategy(Collection> ranges, int nowInSec) + { + this.ranges = ranges; + this.nowInSec = nowInSec; + } + + public static CleanupStrategy get(ColumnFamilyStore cfs, Collection> ranges, int nowInSec) { return cfs.indexManager.hasIndexes() - ? new Full(cfs, ranges) - : new Bounded(cfs, ranges); + ? new Full(cfs, ranges, nowInSec) + : new Bounded(cfs, ranges, nowInSec); } public abstract ISSTableScanner getScanner(SSTableReader sstable, RateLimiter limiter); - public abstract SSTableIdentityIterator cleanup(SSTableIdentityIterator row); + public abstract UnfilteredRowIterator cleanup(UnfilteredRowIterator partition); private static final class Bounded extends CleanupStrategy { - private final Collection> ranges; - - public Bounded(final ColumnFamilyStore cfs, Collection> ranges) + public Bounded(final ColumnFamilyStore cfs, Collection> ranges, int nowInSec) { - this.ranges = ranges; + super(ranges, nowInSec); cacheCleanupExecutor.submit(new Runnable() { @Override @@ -894,8 +896,8 @@ public class CompactionManager implements CompactionManagerMBean cfs.cleanupCache(); } }); - } + @Override public ISSTableScanner getScanner(SSTableReader sstable, RateLimiter limiter) { @@ -903,23 +905,20 @@ public class CompactionManager implements CompactionManagerMBean } @Override - public SSTableIdentityIterator cleanup(SSTableIdentityIterator row) + public UnfilteredRowIterator cleanup(UnfilteredRowIterator partition) { - return row; + return partition; } } private static final class Full extends CleanupStrategy { - private final Collection> ranges; private final ColumnFamilyStore cfs; - private List indexedColumnsInRow; - public Full(ColumnFamilyStore cfs, Collection> ranges) + public Full(ColumnFamilyStore cfs, Collection> ranges, int nowInSec) { + super(ranges, nowInSec); this.cfs = cfs; - this.ranges = ranges; - this.indexedColumnsInRow = null; } @Override @@ -929,36 +928,17 @@ public class CompactionManager implements CompactionManagerMBean } @Override - public SSTableIdentityIterator cleanup(SSTableIdentityIterator row) + public UnfilteredRowIterator cleanup(UnfilteredRowIterator partition) { - if (Range.isInRanges(row.getKey().getToken(), ranges)) - return row; + if (Range.isInRanges(partition.partitionKey().getToken(), ranges)) + return partition; - cfs.invalidateCachedRow(row.getKey()); + cfs.invalidateCachedPartition(partition.partitionKey()); - if (indexedColumnsInRow != null) - indexedColumnsInRow.clear(); - - while (row.hasNext()) + // acquire memtable lock here because secondary index deletion may cause a race. See CASSANDRA-3712 + try (OpOrder.Group opGroup = cfs.keyspace.writeOrder.start()) { - OnDiskAtom column = row.next(); - - if (column instanceof Cell && cfs.indexManager.indexes((Cell) column)) - { - if (indexedColumnsInRow == null) - indexedColumnsInRow = new ArrayList<>(); - - indexedColumnsInRow.add((Cell) column); - } - } - - if (indexedColumnsInRow != null && !indexedColumnsInRow.isEmpty()) - { - // acquire memtable lock here because secondary index deletion may cause a race. See CASSANDRA-3712 - try (OpOrder.Group opGroup = cfs.keyspace.writeOrder.start()) - { - cfs.indexManager.deleteFromIndexes(row.getKey(), indexedColumnsInRow, opGroup); - } + cfs.indexManager.deleteFromIndexes(partition, opGroup, nowInSec); } return null; } @@ -978,14 +958,15 @@ public class CompactionManager implements CompactionManagerMBean expectedBloomFilterSize, repairedAt, sstable.getSSTableLevel(), - cfs.partitioner); + cfs.partitioner, + sstable.header); } public static SSTableWriter createWriterForAntiCompaction(ColumnFamilyStore cfs, - File compactionFileLocation, - int expectedBloomFilterSize, - long repairedAt, - Collection sstables) + File compactionFileLocation, + int expectedBloomFilterSize, + long repairedAt, + Collection sstables) { FileUtils.createDirectory(compactionFileLocation); int minLevel = Integer.MAX_VALUE; @@ -1008,7 +989,8 @@ public class CompactionManager implements CompactionManagerMBean repairedAt, cfs.metadata, cfs.partitioner, - new MetadataCollector(sstables, cfs.metadata.comparator, minLevel)); + new MetadataCollector(sstables, cfs.metadata.comparator, minLevel), + SerializationHeader.make(cfs.metadata, sstables)); } @@ -1033,6 +1015,7 @@ public class CompactionManager implements CompactionManagerMBean String snapshotName = validator.desc.sessionId.toString(); int gcBefore; + int nowInSec = FBUtilities.nowInSeconds(); boolean isSnapshotValidation = cfs.snapshotExists(snapshotName); if (isSnapshotValidation) { @@ -1046,7 +1029,7 @@ public class CompactionManager implements CompactionManagerMBean // this at a different time (that's the whole purpose of repair with snaphsot). So instead we take the creation // time of the snapshot, which should give us roughtly the same time on each replica (roughtly being in that case // 'as good as in the non-snapshot' case) - gcBefore = cfs.gcBefore(cfs.getSnapshotCreationTime(snapshotName)); + gcBefore = cfs.gcBefore((int)(cfs.getSnapshotCreationTime(snapshotName) / 1000)); } else { @@ -1084,7 +1067,7 @@ public class CompactionManager implements CompactionManagerMBean if (validator.gcBefore > 0) gcBefore = validator.gcBefore; else - gcBefore = getDefaultGcBefore(cfs); + gcBefore = getDefaultGcBefore(cfs, nowInSec); } // Create Merkle tree suitable to hold estimated partitions for given range. @@ -1099,32 +1082,28 @@ public class CompactionManager implements CompactionManagerMBean MerkleTree tree = new MerkleTree(cfs.partitioner, validator.desc.range, MerkleTree.RECOMMENDED_DEPTH, (int) Math.pow(2, depth)); long start = System.nanoTime(); - try (AbstractCompactionStrategy.ScannerList scanners = cfs.getCompactionStrategyManager().getScanners(sstables, validator.desc.range)) + try (AbstractCompactionStrategy.ScannerList scanners = cfs.getCompactionStrategyManager().getScanners(sstables, validator.desc.range); + ValidationCompactionController controller = new ValidationCompactionController(cfs, gcBefore); + CompactionIterator ci = new ValidationCompactionIterator(scanners.scanners, controller, nowInSec, metrics)) { - CompactionIterable ci = new ValidationCompactionIterable(cfs, scanners.scanners, gcBefore); - Iterator iter = ci.iterator(); - metrics.beginCompaction(ci); - try + // validate the CF as we iterate over it + validator.prepare(cfs, tree); + while (ci.hasNext()) { - // validate the CF as we iterate over it - validator.prepare(cfs, tree); - while (iter.hasNext()) + if (ci.isStopRequested()) + throw new CompactionInterruptedException(ci.getCompactionInfo()); + try (UnfilteredRowIterator partition = ci.next()) { - if (ci.isStopRequested()) - throw new CompactionInterruptedException(ci.getCompactionInfo()); - AbstractCompactedRow row = iter.next(); - validator.add(row); + validator.add(partition); } - validator.complete(); } - finally + validator.complete(); + } + finally + { + if (isSnapshotValidation) { - if (isSnapshotValidation) - { - cfs.clearSnapshot(snapshotName); - } - - metrics.finishCompaction(ci); + cfs.clearSnapshot(snapshotName); } } @@ -1209,45 +1188,38 @@ public class CompactionManager implements CompactionManagerMBean File destination = cfs.directories.getWriteableLocationAsFile(cfs.getExpectedCompactedFileSize(sstableAsSet, OperationType.ANTICOMPACTION)); long repairedKeyCount = 0; long unrepairedKeyCount = 0; + int nowInSec = FBUtilities.nowInSeconds(); + CompactionStrategyManager strategy = cfs.getCompactionStrategyManager(); try (SSTableRewriter repairedSSTableWriter = new SSTableRewriter(cfs, anticompactionGroup, groupMaxDataAge, false, false); SSTableRewriter unRepairedSSTableWriter = new SSTableRewriter(cfs, anticompactionGroup, groupMaxDataAge, false, false); AbstractCompactionStrategy.ScannerList scanners = strategy.getScanners(anticompactionGroup.originals()); - CompactionController controller = new CompactionController(cfs, sstableAsSet, getDefaultGcBefore(cfs))) + CompactionController controller = new CompactionController(cfs, sstableAsSet, getDefaultGcBefore(cfs, nowInSec)); + CompactionIterator ci = new CompactionIterator(OperationType.ANTICOMPACTION, scanners.scanners, controller, nowInSec, UUIDGen.getTimeUUID(), metrics)) { int expectedBloomFilterSize = Math.max(cfs.metadata.getMinIndexInterval(), (int)(SSTableReader.getApproximateKeyCount(sstableAsSet))); repairedSSTableWriter.switchWriter(CompactionManager.createWriterForAntiCompaction(cfs, destination, expectedBloomFilterSize, repairedAt, sstableAsSet)); unRepairedSSTableWriter.switchWriter(CompactionManager.createWriterForAntiCompaction(cfs, destination, expectedBloomFilterSize, ActiveRepairService.UNREPAIRED_SSTABLE, sstableAsSet)); - CompactionIterable ci = new CompactionIterable(OperationType.ANTICOMPACTION, scanners.scanners, controller, DatabaseDescriptor.getSSTableFormat(), UUIDGen.getTimeUUID()); - metrics.beginCompaction(ci); - try + while (ci.hasNext()) { - @SuppressWarnings("resource") - CloseableIterator iter = ci.iterator(); - while (iter.hasNext()) + try (UnfilteredRowIterator partition = ci.next()) { - @SuppressWarnings("resource") - AbstractCompactedRow row = iter.next(); // if current range from sstable is repaired, save it into the new repaired sstable - if (Range.isInRanges(row.key.getToken(), ranges)) + if (Range.isInRanges(partition.partitionKey().getToken(), ranges)) { - repairedSSTableWriter.append(row); + repairedSSTableWriter.append(partition); repairedKeyCount++; } // otherwise save into the new 'non-repaired' table else { - unRepairedSSTableWriter.append(row); + unRepairedSSTableWriter.append(partition); unrepairedKeyCount++; } } } - finally - { - metrics.finishCompaction(ci); - } List anticompactedSSTables = new ArrayList<>(); // since both writers are operating over the same Transaction, we cannot use the convenience Transactional.finish() method, @@ -1342,19 +1314,18 @@ public class CompactionManager implements CompactionManagerMBean return executor.submit(runnable); } - public static int getDefaultGcBefore(ColumnFamilyStore cfs) + public static int getDefaultGcBefore(ColumnFamilyStore cfs, int nowInSec) { // 2ndary indexes have ExpiringColumns too, so we need to purge tombstones deleted before now. We do not need to // add any GcGrace however since 2ndary indexes are local to a node. - return cfs.isIndex() ? (int) (System.currentTimeMillis() / 1000) : cfs.gcBefore(System.currentTimeMillis()); + return cfs.isIndex() ? nowInSec : cfs.gcBefore(nowInSec); } - private static class ValidationCompactionIterable extends CompactionIterable + private static class ValidationCompactionIterator extends CompactionIterator { - @SuppressWarnings("resource") - public ValidationCompactionIterable(ColumnFamilyStore cfs, List scanners, int gcBefore) + public ValidationCompactionIterator(List scanners, ValidationCompactionController controller, int nowInSec, CompactionMetrics metrics) { - super(OperationType.VALIDATION, scanners, new ValidationCompactionController(cfs, gcBefore), DatabaseDescriptor.getSSTableFormat(), UUIDGen.getTimeUUID()); + super(OperationType.VALIDATION, scanners, controller, nowInSec, UUIDGen.getTimeUUID(), metrics); } } @@ -1518,36 +1489,6 @@ public class CompactionManager implements CompactionManagerMBean return metrics.completedTasks.getValue(); } - private static class CleanupInfo extends CompactionInfo.Holder - { - private final SSTableReader sstable; - private final ISSTableScanner scanner; - private final UUID cleanupCompactionId; - - public CleanupInfo(SSTableReader sstable, ISSTableScanner scanner) - { - this.sstable = sstable; - this.scanner = scanner; - cleanupCompactionId = UUIDGen.getTimeUUID(); - } - - public CompactionInfo getCompactionInfo() - { - try - { - return new CompactionInfo(sstable.metadata, - OperationType.CLEANUP, - scanner.getCurrentPosition(), - scanner.getLengthInBytes(), - cleanupCompactionId); - } - catch (Exception e) - { - throw new RuntimeException(); - } - } - } - public void stopCompaction(String type) { OperationType operation = OperationType.valueOf(type); diff --git a/src/java/org/apache/cassandra/db/compaction/CompactionStrategyManager.java b/src/java/org/apache/cassandra/db/compaction/CompactionStrategyManager.java index dd6261c72d..cfe28e85b3 100644 --- a/src/java/org/apache/cassandra/db/compaction/CompactionStrategyManager.java +++ b/src/java/org/apache/cassandra/db/compaction/CompactionStrategyManager.java @@ -339,11 +339,11 @@ public class CompactionStrategyManager implements INotificationConsumer * @param range * @return */ + @SuppressWarnings("resource") public synchronized AbstractCompactionStrategy.ScannerList getScanners(Collection sstables, Range range) { List repairedSSTables = new ArrayList<>(); List unrepairedSSTables = new ArrayList<>(); - for (SSTableReader sstable : sstables) { if (sstable.isRepaired()) @@ -352,26 +352,26 @@ public class CompactionStrategyManager implements INotificationConsumer unrepairedSSTables.add(sstable); } - List scanners = new ArrayList<>(); - if (!repairedSSTables.isEmpty()) - scanners.addAll(repaired.getScanners(repairedSSTables, range).scanners); - if (!unrepairedSSTables.isEmpty()) - scanners.addAll(unrepaired.getScanners(unrepairedSSTables, range).scanners); + AbstractCompactionStrategy.ScannerList repairedScanners = repaired.getScanners(repairedSSTables, range); + AbstractCompactionStrategy.ScannerList unrepairedScanners = unrepaired.getScanners(unrepairedSSTables, range); + List scanners = new ArrayList<>(repairedScanners.scanners.size() + unrepairedScanners.scanners.size()); + scanners.addAll(repairedScanners.scanners); + scanners.addAll(unrepairedScanners.scanners); return new AbstractCompactionStrategy.ScannerList(scanners); } - public Collection> groupSSTablesForAntiCompaction(Collection sstablesToGroup) - { - return unrepaired.groupSSTablesForAntiCompaction(sstablesToGroup); - } - public synchronized AbstractCompactionStrategy.ScannerList getScanners(Collection sstables) { return getScanners(sstables, null); } + public Collection> groupSSTablesForAntiCompaction(Collection sstablesToGroup) + { + return unrepaired.groupSSTablesForAntiCompaction(sstablesToGroup); + } + public long getMaxSSTableBytes() { return unrepaired.getMaxSSTableBytes(); diff --git a/src/java/org/apache/cassandra/db/compaction/CompactionTask.java b/src/java/org/apache/cassandra/db/compaction/CompactionTask.java index 7dbeb44787..911b4afce3 100644 --- a/src/java/org/apache/cassandra/db/compaction/CompactionTask.java +++ b/src/java/org/apache/cassandra/db/compaction/CompactionTask.java @@ -45,6 +45,7 @@ import org.apache.cassandra.db.compaction.CompactionManager.CompactionExecutorSt import org.apache.cassandra.db.lifecycle.LifecycleTransaction; import org.apache.cassandra.service.ActiveRepairService; import org.apache.cassandra.utils.CloseableIterator; +import org.apache.cassandra.utils.FBUtilities; import org.apache.cassandra.utils.UUIDGen; import org.apache.cassandra.utils.concurrent.Refs; @@ -142,70 +143,63 @@ public class CompactionTask extends AbstractCompactionTask logger.info("Compacting ({}) {}", taskIdLoggerMsg, ssTableLoggerMsg); long start = System.nanoTime(); - long totalKeysWritten = 0; - long estimatedKeys = 0; try (CompactionController controller = getCompactionController(transaction.originals())) { Set actuallyCompact = Sets.difference(transaction.originals(), controller.getFullyExpiredSSTables()); - SSTableFormat.Type sstableFormat = getFormatType(transaction.originals()); - List newSStables; - AbstractCompactionIterable ci; + + long[] mergedRowCounts; // SSTableScanners need to be closed before markCompactedSSTablesReplaced call as scanners contain references // to both ifile and dfile and SSTR will throw deletion errors on Windows if it tries to delete before scanner is closed. // See CASSANDRA-8019 and CASSANDRA-8399 + int nowInSec = FBUtilities.nowInSeconds(); try (Refs refs = Refs.ref(actuallyCompact); - AbstractCompactionStrategy.ScannerList scanners = strategy.getScanners(actuallyCompact)) + AbstractCompactionStrategy.ScannerList scanners = strategy.getScanners(actuallyCompact); + CompactionIterator ci = new CompactionIterator(compactionType, scanners.scanners, controller, nowInSec, taskId)) { + if (collector != null) + collector.beginCompaction(ci); + long lastCheckObsoletion = start; - ci = new CompactionIterable(compactionType, scanners.scanners, controller, sstableFormat, taskId); - try (CloseableIterator iter = ci.iterator()) + if (!controller.cfs.getCompactionStrategyManager().isActive) + throw new CompactionInterruptedException(ci.getCompactionInfo()); + + try (CompactionAwareWriter writer = getCompactionAwareWriter(cfs, transaction, actuallyCompact)) { - if (collector != null) - collector.beginCompaction(ci); - long lastCheckObsoletion = start; - - if (!controller.cfs.getCompactionStrategyManager().isActive) - throw new CompactionInterruptedException(ci.getCompactionInfo()); - - try (CompactionAwareWriter writer = getCompactionAwareWriter(cfs, transaction, actuallyCompact)) + estimatedKeys = writer.estimatedKeys(); + while (ci.hasNext()) { - estimatedKeys = writer.estimatedKeys(); - while (iter.hasNext()) + if (ci.isStopRequested()) + throw new CompactionInterruptedException(ci.getCompactionInfo()); + + if (writer.append(ci.next())) + totalKeysWritten++; + + if (System.nanoTime() - lastCheckObsoletion > TimeUnit.MINUTES.toNanos(1L)) { - if (ci.isStopRequested()) - throw new CompactionInterruptedException(ci.getCompactionInfo()); - - try (AbstractCompactedRow row = iter.next()) - { - if (writer.append(row)) - totalKeysWritten++; - - if (System.nanoTime() - lastCheckObsoletion > TimeUnit.MINUTES.toNanos(1L)) - { - controller.maybeRefreshOverlaps(); - lastCheckObsoletion = System.nanoTime(); - } - } + controller.maybeRefreshOverlaps(); + lastCheckObsoletion = System.nanoTime(); } - - // don't replace old sstables yet, as we need to mark the compaction finished in the system table - newSStables = writer.finish(); } - finally - { - // point of no return -- the new sstables are live on disk; next we'll start deleting the old ones - // (in replaceCompactedSSTables) - if (taskId != null) - SystemKeyspace.finishCompaction(taskId); - if (collector != null) - collector.finishCompaction(ci); - } + // don't replace old sstables yet, as we need to mark the compaction finished in the system table + newSStables = writer.finish(); + } + finally + { + // point of no return -- the new sstables are live on disk; next we'll start deleting the old ones + // (in replaceCompactedSSTables) + if (taskId != null) + SystemKeyspace.finishCompaction(taskId); + + if (collector != null) + collector.finishCompaction(ci); + + mergedRowCounts = ci.getMergedRowCounts(); } } @@ -221,7 +215,7 @@ public class CompactionTask extends AbstractCompactionTask double mbps = dTime > 0 ? (double) endsize / (1024 * 1024) / ((double) dTime / 1000) : 0; long totalSourceRows = 0; - String mergeSummary = updateCompactionHistory(cfs.keyspace.getName(), cfs.getColumnFamilyName(), ci, startsize, endsize); + String mergeSummary = updateCompactionHistory(cfs.keyspace.getName(), cfs.getColumnFamilyName(), mergedRowCounts, startsize, endsize); logger.info(String.format("Compacted (%s) %d sstables to [%s] to level=%d. %,d bytes to %,d (~%d%% of original) in %,dms = %fMB/s. %,d total partitions merged to %,d. Partition merge counts were {%s}", taskIdLoggerMsg, transaction.originals().size(), newSSTableNames.toString(), getLevel(), startsize, endsize, (int) (ratio * 100), dTime, mbps, totalSourceRows, totalKeysWritten, mergeSummary)); logger.debug(String.format("CF Total Bytes Compacted: %,d", CompactionTask.addToTotalBytesCompacted(endsize))); @@ -242,14 +236,13 @@ public class CompactionTask extends AbstractCompactionTask } - public static String updateCompactionHistory(String keyspaceName, String columnFamilyName, AbstractCompactionIterable ci, long startSize, long endSize) + public static String updateCompactionHistory(String keyspaceName, String columnFamilyName, long[] mergedRowCounts, long startSize, long endSize) { - long[] counts = ci.getMergedRowCounts(); - StringBuilder mergeSummary = new StringBuilder(counts.length * 10); + StringBuilder mergeSummary = new StringBuilder(mergedRowCounts.length * 10); Map mergedRows = new HashMap<>(); - for (int i = 0; i < counts.length; i++) + for (int i = 0; i < mergedRowCounts.length; i++) { - long count = counts[i]; + long count = mergedRowCounts[i]; if (count == 0) continue; @@ -305,13 +298,4 @@ public class CompactionTask extends AbstractCompactionTask } return max; } - - public static SSTableFormat.Type getFormatType(Collection sstables) - { - if (sstables.isEmpty() || !SSTableFormat.enableSSTableDevelopmentTestMode) - return DatabaseDescriptor.getSSTableFormat(); - - //Allows us to test compaction of non-default formats - return sstables.iterator().next().descriptor.formatType; - } } diff --git a/src/java/org/apache/cassandra/db/compaction/LazilyCompactedRow.java b/src/java/org/apache/cassandra/db/compaction/LazilyCompactedRow.java deleted file mode 100644 index 93505ae3bc..0000000000 --- a/src/java/org/apache/cassandra/db/compaction/LazilyCompactedRow.java +++ /dev/null @@ -1,346 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.apache.cassandra.db.compaction; - -import java.io.IOException; -import java.nio.ByteBuffer; -import java.security.MessageDigest; -import java.util.Collections; -import java.util.Iterator; -import java.util.List; - -import com.google.common.base.Predicates; -import com.google.common.collect.Iterators; - -import org.apache.cassandra.db.*; -import org.apache.cassandra.db.columniterator.OnDiskAtomIterator; -import org.apache.cassandra.db.index.SecondaryIndexManager; -import org.apache.cassandra.io.sstable.format.big.BigTableWriter; -import org.apache.cassandra.io.sstable.ColumnNameHelper; -import org.apache.cassandra.io.sstable.ColumnStats; -import org.apache.cassandra.io.sstable.SSTable; -import org.apache.cassandra.io.util.DataOutputBuffer; -import org.apache.cassandra.io.util.DataOutputPlus; -import org.apache.cassandra.io.util.SequentialWriter; -import org.apache.cassandra.utils.MergeIterator; -import org.apache.cassandra.utils.StreamingHistogram; -import org.apache.cassandra.utils.Throwables; - -/** - * LazilyCompactedRow only computes the row bloom filter and column index in memory - * (at construction time); it does this by reading one column at a time from each - * of the rows being compacted, and merging them as it does so. So the most we have - * in memory at a time is the bloom filter, the index, and one column from each - * pre-compaction row. - */ -public class LazilyCompactedRow extends AbstractCompactedRow -{ - protected final List rows; - protected final CompactionController controller; - protected boolean hasCalculatedMaxPurgeableTimestamp = false; - protected long maxPurgeableTimestamp; - protected final ColumnFamily emptyColumnFamily; - protected ColumnStats columnStats; - protected boolean closed; - protected ColumnIndex.Builder indexBuilder; - protected final SecondaryIndexManager.Updater indexer; - protected final Reducer reducer; - protected final Iterator merger; - protected DeletionTime maxRowTombstone; - - public LazilyCompactedRow(CompactionController controller, List rows) - { - super(rows.get(0).getKey()); - this.rows = rows; - this.controller = controller; - indexer = controller.cfs.indexManager.gcUpdaterFor(key); - - // Combine top-level tombstones, keeping the one with the highest markedForDeleteAt timestamp. This may be - // purged (depending on gcBefore), but we need to remember it to properly delete columns during the merge - maxRowTombstone = DeletionTime.LIVE; - for (OnDiskAtomIterator row : rows) - { - DeletionTime rowTombstone = row.getColumnFamily().deletionInfo().getTopLevelDeletion(); - if (maxRowTombstone.compareTo(rowTombstone) < 0) - maxRowTombstone = rowTombstone; - } - - emptyColumnFamily = ArrayBackedSortedColumns.factory.create(controller.cfs.metadata); - emptyColumnFamily.delete(maxRowTombstone); - if (!maxRowTombstone.isLive() && maxRowTombstone.markedForDeleteAt < getMaxPurgeableTimestamp()) - emptyColumnFamily.purgeTombstones(controller.gcBefore); - - reducer = new Reducer(); - merger = Iterators.filter(MergeIterator.get(rows, emptyColumnFamily.getComparator().onDiskAtomComparator(), reducer), Predicates.notNull()); - } - - /** - * tombstones with a localDeletionTime before this can be purged. This is the minimum timestamp for any sstable - * containing `key` outside of the set of sstables involved in this compaction. - */ - private long getMaxPurgeableTimestamp() - { - if (!hasCalculatedMaxPurgeableTimestamp) - { - hasCalculatedMaxPurgeableTimestamp = true; - maxPurgeableTimestamp = controller.maxPurgeableTimestamp(key); - } - return maxPurgeableTimestamp; - } - - private static void removeDeleted(ColumnFamily cf, boolean shouldPurge, DecoratedKey key, CompactionController controller) - { - // We should only purge cell tombstones if shouldPurge is true, but regardless, it's still ok to remove cells that - // are shadowed by a row or range tombstone; removeDeletedColumnsOnly(cf, Integer.MIN_VALUE) will accomplish this - // without purging tombstones. - int overriddenGCBefore = shouldPurge ? controller.gcBefore : Integer.MIN_VALUE; - ColumnFamilyStore.removeDeletedColumnsOnly(cf, overriddenGCBefore, controller.cfs.indexManager.gcUpdaterFor(key)); - } - - public RowIndexEntry write(long currentPosition, SequentialWriter dataFile) throws IOException - { - assert !closed; - - DataOutputPlus out = dataFile.stream; - - ColumnIndex columnsIndex; - try - { - indexBuilder = new ColumnIndex.Builder(emptyColumnFamily, key.getKey(), out); - columnsIndex = indexBuilder.buildForCompaction(merger); - - // if there aren't any columns or tombstones, return null - if (columnsIndex.columnsIndex.isEmpty() && !emptyColumnFamily.isMarkedForDelete()) - return null; - } - catch (IOException e) - { - throw new RuntimeException(e); - } - // reach into the reducer (created during iteration) to get column count, size, max column timestamp - columnStats = new ColumnStats(reducer.columns, - reducer.minTimestampTracker.get(), - Math.max(emptyColumnFamily.deletionInfo().maxTimestamp(), reducer.maxTimestampTracker.get()), - reducer.maxDeletionTimeTracker.get(), - reducer.tombstones, - reducer.minColumnNameSeen, - reducer.maxColumnNameSeen, - reducer.hasLegacyCounterShards); - - // in case no columns were ever written, we may still need to write an empty header with a top-level tombstone - indexBuilder.maybeWriteEmptyRowHeader(); - - out.writeShort(BigTableWriter.END_OF_ROW); - - close(); - - return RowIndexEntry.create(currentPosition, emptyColumnFamily.deletionInfo().getTopLevelDeletion(), columnsIndex); - } - - public void update(MessageDigest digest) - { - assert !closed; - - // no special-case for rows.size == 1, we're actually skipping some bytes here so just - // blindly updating everything wouldn't be correct - try (DataOutputBuffer out = new DataOutputBuffer()) - { - // initialize indexBuilder for the benefit of its tombstoneTracker, used by our reducing iterator - indexBuilder = new ColumnIndex.Builder(emptyColumnFamily, key.getKey(), out); - - DeletionTime.serializer.serialize(emptyColumnFamily.deletionInfo().getTopLevelDeletion(), out); - - // do not update digest in case of missing or purged row level tombstones, see CASSANDRA-8979 - // - digest for non-empty rows needs to be updated with deletion in any case to match digest with versions before patch - // - empty rows must not update digest in case of LIVE delete status to avoid mismatches with non-existing rows - // this will however introduce in return a digest mismatch for versions before patch (which would update digest in any case) - if (merger.hasNext() || emptyColumnFamily.deletionInfo().getTopLevelDeletion() != DeletionTime.LIVE) - { - digest.update(out.getData(), 0, out.getLength()); - } - } - catch (IOException e) - { - throw new AssertionError(e); - } - - while (merger.hasNext()) - merger.next().updateDigest(digest); - close(); - } - - public ColumnStats columnStats() - { - return columnStats; - } - - public void close() - { - Throwable accumulate = null; - for (OnDiskAtomIterator row : rows) - { - try - { - row.close(); - } - catch (IOException e) - { - accumulate = Throwables.merge(accumulate, e); - } - } - closed = true; - Throwables.maybeFail(accumulate); - } - - protected class Reducer extends MergeIterator.Reducer - { - // all columns reduced together will have the same name, so there will only be one column - // in the container; we just want to leverage the conflict resolution code from CF. - // (Note that we add the row tombstone in getReduced.) - ColumnFamily container = ArrayBackedSortedColumns.factory.create(emptyColumnFamily.metadata()); - - // tombstone reference; will be reconciled w/ column during getReduced. Note that the top-level (row) tombstone - // is held by LCR.deletionInfo. - public RangeTombstone tombstone; - - public int columns = 0; - // if the row tombstone is 'live' we need to set timestamp to MAX_VALUE to be able to overwrite it later - // markedForDeleteAt is MIN_VALUE for 'live' row tombstones (which we use to default maxTimestampSeen) - - ColumnStats.MinLongTracker minTimestampTracker = new ColumnStats.MinLongTracker(Long.MIN_VALUE); - ColumnStats.MaxLongTracker maxTimestampTracker = new ColumnStats.MaxLongTracker(Long.MAX_VALUE); - // we need to set MIN_VALUE if we are 'live' since we want to overwrite it later - // we are bound to have either a RangeTombstone or standard cells will set this properly: - ColumnStats.MaxIntTracker maxDeletionTimeTracker = new ColumnStats.MaxIntTracker(Integer.MAX_VALUE); - - public StreamingHistogram tombstones = new StreamingHistogram(SSTable.TOMBSTONE_HISTOGRAM_BIN_SIZE); - public List minColumnNameSeen = Collections.emptyList(); - public List maxColumnNameSeen = Collections.emptyList(); - public boolean hasLegacyCounterShards = false; - - public Reducer() - { - minTimestampTracker.update(maxRowTombstone.isLive() ? Long.MAX_VALUE : maxRowTombstone.markedForDeleteAt); - maxTimestampTracker.update(maxRowTombstone.markedForDeleteAt); - maxDeletionTimeTracker.update(maxRowTombstone.isLive() ? Integer.MIN_VALUE : maxRowTombstone.localDeletionTime); - } - - /** - * Called once per version of a cell that we need to merge, after which getReduced() is called. In other words, - * this will be called one or more times with cells that share the same column name. - */ - public void reduce(OnDiskAtom current) - { - if (current instanceof RangeTombstone) - { - if (tombstone == null || current.timestamp() >= tombstone.timestamp()) - tombstone = (RangeTombstone)current; - } - else - { - Cell cell = (Cell) current; - container.addColumn(cell); - - // skip the index-update checks if there is no indexing needed since they are a bit expensive - if (indexer == SecondaryIndexManager.nullUpdater) - return; - - if (cell.isLive() && !container.getColumn(cell.name()).equals(cell)) - indexer.remove(cell); - } - } - - /** - * Called after reduce() has been called for each cell sharing the same name. - */ - protected OnDiskAtom getReduced() - { - if (tombstone != null) - { - RangeTombstone t = tombstone; - tombstone = null; - - if (t.timestamp() < getMaxPurgeableTimestamp() && t.data.isGcAble(controller.gcBefore)) - { - indexBuilder.tombstoneTracker().update(t, true); - return null; - } - else - { - tombstones.update(t.getLocalDeletionTime()); - minTimestampTracker.update(t.timestamp()); - maxTimestampTracker.update(t.timestamp()); - maxDeletionTimeTracker.update(t.getLocalDeletionTime()); - minColumnNameSeen = ColumnNameHelper.minComponents(minColumnNameSeen, t.min, controller.cfs.metadata.comparator); - maxColumnNameSeen = ColumnNameHelper.maxComponents(maxColumnNameSeen, t.max, controller.cfs.metadata.comparator); - return t; - } - } - else - { - // when we clear() the container, it removes the deletion info, so this needs to be reset each time - container.delete(maxRowTombstone); - Iterator iter = container.iterator(); - Cell c = iter.next(); - boolean shouldPurge = c.getLocalDeletionTime() < Integer.MAX_VALUE && c.timestamp() < getMaxPurgeableTimestamp(); - removeDeleted(container, shouldPurge, key, controller); - iter = container.iterator(); - if (!iter.hasNext()) - { - // don't call clear() because that resets the deletion time. See CASSANDRA-7808. - container = ArrayBackedSortedColumns.factory.create(emptyColumnFamily.metadata()); - return null; - } - - int localDeletionTime = container.deletionInfo().getTopLevelDeletion().localDeletionTime; - if (localDeletionTime < Integer.MAX_VALUE) - tombstones.update(localDeletionTime); - - Cell reduced = iter.next(); - container = ArrayBackedSortedColumns.factory.create(emptyColumnFamily.metadata()); - - // removeDeleted have only checked the top-level CF deletion times, - // not the range tombstone. For that we use the columnIndexer tombstone tracker. - if (indexBuilder.tombstoneTracker().isDeleted(reduced)) - { - // We skip that column so it won't be passed to the tracker by the index builded. So pass it now to - // make sure we still discard potentially un-needed RT as soon as possible. - indexBuilder.tombstoneTracker().update(reduced, false); - indexer.remove(reduced); - return null; - } - - columns++; - minTimestampTracker.update(reduced.timestamp()); - maxTimestampTracker.update(reduced.timestamp()); - maxDeletionTimeTracker.update(reduced.getLocalDeletionTime()); - minColumnNameSeen = ColumnNameHelper.minComponents(minColumnNameSeen, reduced.name(), controller.cfs.metadata.comparator); - maxColumnNameSeen = ColumnNameHelper.maxComponents(maxColumnNameSeen, reduced.name(), controller.cfs.metadata.comparator); - - int deletionTime = reduced.getLocalDeletionTime(); - if (deletionTime < Integer.MAX_VALUE) - tombstones.update(deletionTime); - - if (reduced instanceof CounterCell) - hasLegacyCounterShards = hasLegacyCounterShards || ((CounterCell) reduced).hasLegacyShards(); - - return reduced; - } - } - } -} diff --git a/src/java/org/apache/cassandra/db/compaction/LeveledCompactionStrategy.java b/src/java/org/apache/cassandra/db/compaction/LeveledCompactionStrategy.java index 9eb58ff3df..dcdfd7f208 100644 --- a/src/java/org/apache/cassandra/db/compaction/LeveledCompactionStrategy.java +++ b/src/java/org/apache/cassandra/db/compaction/LeveledCompactionStrategy.java @@ -17,7 +17,6 @@ */ package org.apache.cassandra.db.compaction; -import java.io.IOException; import java.util.*; @@ -26,17 +25,18 @@ import com.google.common.base.Joiner; import com.google.common.collect.*; import com.google.common.primitives.Doubles; -import org.apache.cassandra.io.sstable.ISSTableScanner; -import org.apache.cassandra.io.sstable.format.SSTableReader; import org.slf4j.Logger; import org.slf4j.LoggerFactory; import org.apache.cassandra.db.ColumnFamilyStore; -import org.apache.cassandra.db.columniterator.OnDiskAtomIterator; import org.apache.cassandra.db.lifecycle.LifecycleTransaction; +import org.apache.cassandra.db.rows.UnfilteredRowIterator; import org.apache.cassandra.dht.Range; import org.apache.cassandra.dht.Token; import org.apache.cassandra.exceptions.ConfigurationException; +import org.apache.cassandra.io.sstable.ISSTableScanner; +import org.apache.cassandra.io.sstable.format.SSTableReader; +import org.apache.cassandra.utils.FBUtilities; public class LeveledCompactionStrategy extends AbstractCompactionStrategy { @@ -244,7 +244,11 @@ public class LeveledCompactionStrategy extends AbstractCompactionStrategy // Create a LeveledScanner that only opens one sstable at a time, in sorted order List intersecting = LeveledScanner.intersecting(byLevel.get(level), range); if (!intersecting.isEmpty()) - scanners.add(new LeveledScanner(intersecting, range)); + { + @SuppressWarnings("resource") // The ScannerList will be in charge of closing (and we close properly on errors) + ISSTableScanner scanner = new LeveledScanner(intersecting, range); + scanners.add(scanner); + } } } } @@ -284,7 +288,7 @@ public class LeveledCompactionStrategy extends AbstractCompactionStrategy // Lazily creates SSTableBoundedScanner for sstable that are assumed to be from the // same level (e.g. non overlapping) - see #4142 - private static class LeveledScanner extends AbstractIterator implements ISSTableScanner + private static class LeveledScanner extends AbstractIterator implements ISSTableScanner { private final Range range; private final List sstables; @@ -332,36 +336,34 @@ public class LeveledCompactionStrategy extends AbstractCompactionStrategy return filtered; } - protected OnDiskAtomIterator computeNext() + public boolean isForThrift() + { + return false; + } + + protected UnfilteredRowIterator computeNext() { if (currentScanner == null) return endOfData(); - try + while (true) { - while (true) - { - if (currentScanner.hasNext()) - return currentScanner.next(); + if (currentScanner.hasNext()) + return currentScanner.next(); - positionOffset += currentScanner.getLengthInBytes(); - currentScanner.close(); - if (!sstableIterator.hasNext()) - { - // reset to null so getCurrentPosition does not return wrong value - currentScanner = null; - return endOfData(); - } - currentScanner = sstableIterator.next().getScanner(range, CompactionManager.instance.getRateLimiter()); + positionOffset += currentScanner.getLengthInBytes(); + currentScanner.close(); + if (!sstableIterator.hasNext()) + { + // reset to null so getCurrentPosition does not return wrong value + currentScanner = null; + return endOfData(); } - } - catch (IOException e) - { - throw new RuntimeException(e); + currentScanner = sstableIterator.next().getScanner(range, CompactionManager.instance.getRateLimiter()); } } - public void close() throws IOException + public void close() { if (currentScanner != null) currentScanner.close(); diff --git a/src/java/org/apache/cassandra/db/compaction/LeveledManifest.java b/src/java/org/apache/cassandra/db/compaction/LeveledManifest.java index 076331646d..0cee370561 100644 --- a/src/java/org/apache/cassandra/db/compaction/LeveledManifest.java +++ b/src/java/org/apache/cassandra/db/compaction/LeveledManifest.java @@ -27,13 +27,14 @@ import com.google.common.collect.ImmutableSortedSet; import com.google.common.collect.Iterables; import com.google.common.collect.Sets; import com.google.common.primitives.Ints; + +import org.apache.cassandra.db.PartitionPosition; import org.apache.cassandra.io.sstable.format.SSTableReader; import org.slf4j.Logger; import org.slf4j.LoggerFactory; import org.apache.cassandra.config.DatabaseDescriptor; import org.apache.cassandra.db.ColumnFamilyStore; -import org.apache.cassandra.db.RowPosition; import org.apache.cassandra.dht.Bounds; import org.apache.cassandra.dht.Range; import org.apache.cassandra.dht.Token; @@ -63,7 +64,7 @@ public class LeveledManifest private final ColumnFamilyStore cfs; @VisibleForTesting protected final List[] generations; - private final RowPosition[] lastCompactedKeys; + private final PartitionPosition[] lastCompactedKeys; private final long maxSSTableSizeInBytes; private final SizeTieredCompactionStrategyOptions options; private final int [] compactionCounter; @@ -75,7 +76,7 @@ public class LeveledManifest this.options = options; generations = new List[MAX_LEVEL_COUNT]; - lastCompactedKeys = new RowPosition[MAX_LEVEL_COUNT]; + lastCompactedKeys = new PartitionPosition[MAX_LEVEL_COUNT]; for (int i = 0; i < generations.length; i++) { generations[i] = new ArrayList<>(); @@ -402,8 +403,8 @@ public class LeveledManifest // say we are compacting 3 sstables: 0->30 in L1 and 0->12, 12->33 in L2 // this means that we will not create overlap in L2 if we add an sstable // contained within 0 -> 33 to the compaction - RowPosition max = null; - RowPosition min = null; + PartitionPosition max = null; + PartitionPosition min = null; for (SSTableReader candidate : candidates) { if (min == null || candidate.first.compareTo(min) < 0) @@ -414,10 +415,10 @@ public class LeveledManifest if (min == null || max == null || min.equals(max)) // single partition sstables - we cannot include a high level sstable. return candidates; Set compacting = cfs.getTracker().getCompacting(); - Range boundaries = new Range<>(min, max); + Range boundaries = new Range<>(min, max); for (SSTableReader sstable : getLevel(i)) { - Range r = new Range(sstable.first, sstable.last); + Range r = new Range(sstable.first, sstable.last); if (boundaries.contains(r) && !compacting.contains(sstable)) { logger.info("Adding high-level (L{}) {} to candidates", sstable.getSSTableLevel(), sstable); @@ -546,8 +547,8 @@ public class LeveledManifest { Set compactingL0 = getCompacting(0); - RowPosition lastCompactingKey = null; - RowPosition firstCompactingKey = null; + PartitionPosition lastCompactingKey = null; + PartitionPosition firstCompactingKey = null; for (SSTableReader candidate : compactingL0) { if (firstCompactingKey == null || candidate.first.compareTo(firstCompactingKey) < 0) diff --git a/src/java/org/apache/cassandra/db/compaction/Scrubber.java b/src/java/org/apache/cassandra/db/compaction/Scrubber.java index 10952e7b2d..562d6810a2 100644 --- a/src/java/org/apache/cassandra/db/compaction/Scrubber.java +++ b/src/java/org/apache/cassandra/db/compaction/Scrubber.java @@ -26,6 +26,8 @@ import com.google.common.base.Throwables; import org.apache.cassandra.db.*; import org.apache.cassandra.db.lifecycle.LifecycleTransaction; +import org.apache.cassandra.db.rows.*; +import org.apache.cassandra.db.partitions.*; import org.apache.cassandra.io.sstable.*; import org.apache.cassandra.io.sstable.format.SSTableReader; import org.apache.cassandra.io.sstable.format.SSTableWriter; @@ -33,6 +35,7 @@ import org.apache.cassandra.io.util.FileUtils; import org.apache.cassandra.io.util.RandomAccessReader; import org.apache.cassandra.service.ActiveRepairService; import org.apache.cassandra.utils.ByteBufferUtil; +import org.apache.cassandra.utils.FBUtilities; import org.apache.cassandra.utils.JVMStabilityInspector; import org.apache.cassandra.utils.OutputHandler; import org.apache.cassandra.utils.UUIDGen; @@ -45,7 +48,6 @@ public class Scrubber implements Closeable private final File destination; private final boolean skipCorrupted; - private final CompactionController controller; private final boolean isCommutative; private final boolean isIndex; private final boolean checkData; @@ -72,14 +74,14 @@ public class Scrubber implements Closeable private final OutputHandler outputHandler; - private static final Comparator rowComparator = new Comparator() + private static final Comparator partitionComparator = new Comparator() { - public int compare(Row r1, Row r2) + public int compare(Partition r1, Partition r2) { - return r1.key.compareTo(r2.key); + return r1.partitionKey().compareTo(r2.partitionKey()); } }; - private final SortedSet outOfOrderRows = new TreeSet<>(rowComparator); + private final SortedSet outOfOrder = new TreeSet<>(partitionComparator); public Scrubber(ColumnFamilyStore cfs, LifecycleTransaction transaction, boolean skipCorrupted, boolean isOffline, boolean checkData) throws IOException { @@ -95,7 +97,9 @@ public class Scrubber implements Closeable this.outputHandler = outputHandler; this.skipCorrupted = skipCorrupted; this.isOffline = isOffline; - this.rowIndexEntrySerializer = sstable.descriptor.version.getSSTableFormat().getIndexSerializer(sstable.metadata); + this.rowIndexEntrySerializer = sstable.descriptor.version.getSSTableFormat().getIndexSerializer(sstable.metadata, + sstable.descriptor.version, + sstable.header); List toScrub = Collections.singletonList(sstable); @@ -104,10 +108,6 @@ public class Scrubber implements Closeable if (destination == null) throw new IOException("disk full"); - // If we run scrub offline, we should never purge tombstone, as we cannot know if other sstable have data that the tombstone deletes. - this.controller = isOffline - ? new ScrubController(cfs) - : new CompactionController(cfs, Collections.singleton(sstable), CompactionManager.getDefaultGcBefore(cfs)); this.isCommutative = cfs.metadata.isCounter(); this.isIndex = cfs.isIndex(); this.checkData = checkData && !this.isIndex; //LocalByPartitionerType does not support validation @@ -127,15 +127,21 @@ public class Scrubber implements Closeable this.nextRowPositionFromIndex = 0; } + private UnfilteredRowIterator withValidation(UnfilteredRowIterator iter, String filename) + { + return checkData ? UnfilteredRowIterators.withValidation(iter, filename) : iter; + } + public void scrub() { outputHandler.output(String.format("Scrubbing %s (%s bytes)", sstable, dataFile.length())); + int nowInSec = FBUtilities.nowInSeconds(); try (SSTableRewriter writer = new SSTableRewriter(cfs, transaction, sstable.maxDataAge, isOffline);) { nextIndexKey = ByteBufferUtil.readWithShortLength(indexFile); { // throw away variable so we don't have a side effect in the assert - long firstRowPositionFromIndex = rowIndexEntrySerializer.deserialize(indexFile, sstable.descriptor.version).position; + long firstRowPositionFromIndex = rowIndexEntrySerializer.deserialize(indexFile).position; assert firstRowPositionFromIndex == 0 : firstRowPositionFromIndex; } @@ -188,7 +194,8 @@ public class Scrubber implements Closeable if (currentIndexKey != null && !key.getKey().equals(currentIndexKey)) { throw new IOError(new IOException(String.format("Key from data file (%s) does not match key from index file (%s)", - ByteBufferUtil.bytesToHex(key.getKey()), ByteBufferUtil.bytesToHex(currentIndexKey)))); + //ByteBufferUtil.bytesToHex(key.getKey()), ByteBufferUtil.bytesToHex(currentIndexKey)))); + "_too big_", ByteBufferUtil.bytesToHex(currentIndexKey)))); } if (dataSizeFromIndex > dataFile.length()) @@ -197,20 +204,19 @@ public class Scrubber implements Closeable if (dataStart != dataStartFromIndex) outputHandler.warn(String.format("Data file row position %d differs from index file row position %d", dataStart, dataStartFromIndex)); - SSTableIdentityIterator atoms = new SSTableIdentityIterator(sstable, dataFile, key, checkData); - - if (prevKey != null && prevKey.compareTo(key) > 0) + try (UnfilteredRowIterator iterator = withValidation(new SSTableIdentityIterator(sstable, dataFile, key), dataFile.getPath())) { - saveOutOfOrderRow(prevKey, key, atoms); - continue; - } + if (prevKey != null && prevKey.compareTo(key) > 0) + { + saveOutOfOrderRow(prevKey, key, iterator); + continue; + } - @SuppressWarnings("resource") - AbstractCompactedRow compactedRow = new LazilyCompactedRow(controller, Collections.singletonList(atoms)); - if (writer.tryAppend(compactedRow) == null) - emptyRows++; - else - goodRows++; + if (writer.tryAppend(iterator) == null) + emptyRows++; + else + goodRows++; + } prevKey = key; } @@ -229,19 +235,19 @@ public class Scrubber implements Closeable { dataFile.seek(dataStartFromIndex); - SSTableIdentityIterator atoms = new SSTableIdentityIterator(sstable, dataFile, key, checkData); - if (prevKey != null && prevKey.compareTo(key) > 0) + try (UnfilteredRowIterator iterator = withValidation(new SSTableIdentityIterator(sstable, dataFile, key), dataFile.getPath())) { - saveOutOfOrderRow(prevKey, key, atoms); - continue; - } + if (prevKey != null && prevKey.compareTo(key) > 0) + { + saveOutOfOrderRow(prevKey, key, iterator); + continue; + } - @SuppressWarnings("resource") - AbstractCompactedRow compactedRow = new LazilyCompactedRow(controller, Collections.singletonList(atoms)); - if (writer.tryAppend(compactedRow) == null) - emptyRows++; - else - goodRows++; + if (writer.tryAppend(iterator) == null) + emptyRows++; + else + goodRows++; + } prevKey = key; } @@ -267,18 +273,18 @@ public class Scrubber implements Closeable } } - if (!outOfOrderRows.isEmpty()) + if (!outOfOrder.isEmpty()) { // out of order rows, but no bad rows found - we can keep our repairedAt time long repairedAt = badRows > 0 ? ActiveRepairService.UNREPAIRED_SSTABLE : sstable.getSSTableMetadata().repairedAt; try (SSTableWriter inOrderWriter = CompactionManager.createWriter(cfs, destination, expectedBloomFilterSize, repairedAt, sstable);) { - for (Row row : outOfOrderRows) - inOrderWriter.append(row.key, row.cf); + for (Partition partition : outOfOrder) + inOrderWriter.append(partition.unfilteredIterator()); newInOrderSstable = inOrderWriter.finish(-1, sstable.maxDataAge, true); } transaction.update(newInOrderSstable, false); - outputHandler.warn(String.format("%d out of order rows found while scrubbing %s; Those have been written (in order) to a new sstable (%s)", outOfOrderRows.size(), sstable, newInOrderSstable)); + outputHandler.warn(String.format("%d out of order rows found while scrubbing %s; Those have been written (in order) to a new sstable (%s)", outOfOrder.size(), sstable, newInOrderSstable)); } // finish obsoletes the old sstable @@ -290,10 +296,6 @@ public class Scrubber implements Closeable { throw Throwables.propagate(e); } - finally - { - controller.close(); - } if (newSstable == null) { @@ -318,8 +320,8 @@ public class Scrubber implements Closeable { nextIndexKey = indexFile.isEOF() ? null : ByteBufferUtil.readWithShortLength(indexFile); nextRowPositionFromIndex = indexFile.isEOF() - ? dataFile.length() - : rowIndexEntrySerializer.deserialize(indexFile, sstable.descriptor.version).position; + ? dataFile.length() + : rowIndexEntrySerializer.deserialize(indexFile).position; } catch (Throwable th) { @@ -350,19 +352,11 @@ public class Scrubber implements Closeable } } - private void saveOutOfOrderRow(DecoratedKey prevKey, DecoratedKey key, SSTableIdentityIterator atoms) + private void saveOutOfOrderRow(DecoratedKey prevKey, DecoratedKey key, UnfilteredRowIterator iterator) { // TODO bitch if the row is too large? if it is there's not much we can do ... outputHandler.warn(String.format("Out of order row detected (%s found after %s)", key, prevKey)); - // adding atoms in sorted order is worst-case for TMBSC, but we shouldn't need to do this very often - // and there's no sense in failing on mis-sorted cells when a TreeMap could safe us - ColumnFamily cf = atoms.getColumnFamily().cloneMeShallow(ArrayBackedSortedColumns.factory, false); - while (atoms.hasNext()) - { - OnDiskAtom atom = atoms.next(); - cf.addAtom(atom); - } - outOfOrderRows.add(new Row(key, cf)); + outOfOrder.add(ArrayBackedPartition.create(iterator)); } public SSTableReader getNewSSTable() @@ -442,20 +436,6 @@ public class Scrubber implements Closeable } } - private static class ScrubController extends CompactionController - { - public ScrubController(ColumnFamilyStore cfs) - { - super(cfs, Integer.MAX_VALUE); - } - - @Override - public long maxPurgeableTimestamp(DecoratedKey key) - { - return Long.MIN_VALUE; - } - } - @VisibleForTesting public ScrubResult scrubWithResult() { diff --git a/src/java/org/apache/cassandra/db/compaction/Upgrader.java b/src/java/org/apache/cassandra/db/compaction/Upgrader.java index a0cce2460d..e3764c8a7b 100644 --- a/src/java/org/apache/cassandra/db/compaction/Upgrader.java +++ b/src/java/org/apache/cassandra/db/compaction/Upgrader.java @@ -21,16 +21,18 @@ import java.io.File; import java.util.*; import com.google.common.base.Throwables; +import com.google.common.collect.Sets; import org.apache.cassandra.config.DatabaseDescriptor; import org.apache.cassandra.db.ColumnFamilyStore; import org.apache.cassandra.db.DecoratedKey; import org.apache.cassandra.db.lifecycle.LifecycleTransaction; +import org.apache.cassandra.db.SerializationHeader; import org.apache.cassandra.io.sstable.*; import org.apache.cassandra.io.sstable.format.SSTableReader; import org.apache.cassandra.io.sstable.format.SSTableWriter; import org.apache.cassandra.io.sstable.metadata.MetadataCollector; -import org.apache.cassandra.utils.CloseableIterator; +import org.apache.cassandra.utils.FBUtilities; import org.apache.cassandra.utils.OutputHandler; import org.apache.cassandra.utils.UUIDGen; @@ -78,24 +80,27 @@ public class Upgrader sstableMetadataCollector.addAncestor(i); } sstableMetadataCollector.sstableLevel(sstable.getSSTableLevel()); - return SSTableWriter.create(Descriptor.fromFilename(cfs.getTempSSTablePath(directory)), estimatedRows, repairedAt, cfs.metadata, cfs.partitioner, sstableMetadataCollector); + return SSTableWriter.create(Descriptor.fromFilename(cfs.getTempSSTablePath(directory)), + estimatedRows, + repairedAt, + cfs.metadata, + cfs.partitioner, + sstableMetadataCollector, + SerializationHeader.make(cfs.metadata, Sets.newHashSet(sstable))); } public void upgrade() { outputHandler.output("Upgrading " + sstable); + int nowInSec = FBUtilities.nowInSeconds(); try (SSTableRewriter writer = new SSTableRewriter(cfs, transaction, CompactionTask.getMaxDataAge(transaction.originals()), true); AbstractCompactionStrategy.ScannerList scanners = strategyManager.getScanners(transaction.originals()); - CloseableIterator iter = new CompactionIterable(compactionType, scanners.scanners, controller, DatabaseDescriptor.getSSTableFormat(), UUIDGen.getTimeUUID()).iterator()) + CompactionIterator iter = new CompactionIterator(compactionType, scanners.scanners, controller, nowInSec, UUIDGen.getTimeUUID())) { writer.switchWriter(createCompactionWriter(sstable.getSSTableMetadata().repairedAt)); while (iter.hasNext()) - { - @SuppressWarnings("resource") - AbstractCompactedRow row = iter.next(); - writer.append(row); - } + writer.append(iter.next()); writer.finish(); outputHandler.output("Upgrade of " + sstable + " complete."); diff --git a/src/java/org/apache/cassandra/db/compaction/Verifier.java b/src/java/org/apache/cassandra/db/compaction/Verifier.java index 0177819441..90a97a08d1 100644 --- a/src/java/org/apache/cassandra/db/compaction/Verifier.java +++ b/src/java/org/apache/cassandra/db/compaction/Verifier.java @@ -18,8 +18,9 @@ package org.apache.cassandra.db.compaction; import com.google.common.base.Throwables; -import com.google.common.collect.Sets; + import org.apache.cassandra.db.*; +import org.apache.cassandra.db.rows.UnfilteredRowIterator; import org.apache.cassandra.io.sstable.Component; import org.apache.cassandra.io.sstable.CorruptSSTableException; @@ -31,7 +32,7 @@ import org.apache.cassandra.io.util.FileUtils; import org.apache.cassandra.io.util.RandomAccessReader; import org.apache.cassandra.service.ActiveRepairService; import org.apache.cassandra.utils.ByteBufferUtil; -import org.apache.cassandra.utils.JVMStabilityInspector; +import org.apache.cassandra.utils.FBUtilities; import org.apache.cassandra.utils.OutputHandler; import org.apache.cassandra.utils.UUIDGen; @@ -71,7 +72,7 @@ public class Verifier implements Closeable this.cfs = cfs; this.sstable = sstable; this.outputHandler = outputHandler; - this.rowIndexEntrySerializer = sstable.descriptor.version.getSSTableFormat().getIndexSerializer(sstable.metadata); + this.rowIndexEntrySerializer = sstable.descriptor.version.getSSTableFormat().getIndexSerializer(sstable.metadata, sstable.descriptor.version, sstable.header); this.controller = new VerifyController(cfs); @@ -102,7 +103,7 @@ public class Verifier implements Closeable } else { - outputHandler.output("Data digest missing, assuming extended verification of disk atoms"); + outputHandler.output("Data digest missing, assuming extended verification of disk values"); extended = true; } } @@ -119,14 +120,14 @@ public class Verifier implements Closeable if ( !extended ) return; - outputHandler.output("Extended Verify requested, proceeding to inspect atoms"); + outputHandler.output("Extended Verify requested, proceeding to inspect values"); try { ByteBuffer nextIndexKey = ByteBufferUtil.readWithShortLength(indexFile); { - long firstRowPositionFromIndex = rowIndexEntrySerializer.deserialize(indexFile, sstable.descriptor.version).position; + long firstRowPositionFromIndex = rowIndexEntrySerializer.deserialize(indexFile).position; if (firstRowPositionFromIndex != 0) markAndThrow(); } @@ -160,7 +161,7 @@ public class Verifier implements Closeable nextIndexKey = indexFile.isEOF() ? null : ByteBufferUtil.readWithShortLength(indexFile); nextRowPositionFromIndex = indexFile.isEOF() ? dataFile.length() - : rowIndexEntrySerializer.deserialize(indexFile, sstable.descriptor.version).position; + : rowIndexEntrySerializer.deserialize(indexFile).position; } catch (Throwable th) { @@ -185,7 +186,10 @@ public class Verifier implements Closeable markAndThrow(); //mimic the scrub read path - new SSTableIdentityIterator(sstable, dataFile, key, true); + try (UnfilteredRowIterator iterator = new SSTableIdentityIterator(sstable, dataFile, key)) + { + } + if ( (prevKey != null && prevKey.compareTo(key) > 0) || !key.getKey().equals(currentIndexKey) || dataStart != dataStartFromIndex ) markAndThrow(); diff --git a/src/java/org/apache/cassandra/db/compaction/writers/CompactionAwareWriter.java b/src/java/org/apache/cassandra/db/compaction/writers/CompactionAwareWriter.java index 20c96d66e2..610592fe92 100644 --- a/src/java/org/apache/cassandra/db/compaction/writers/CompactionAwareWriter.java +++ b/src/java/org/apache/cassandra/db/compaction/writers/CompactionAwareWriter.java @@ -23,7 +23,7 @@ import java.util.Set; import org.apache.cassandra.db.ColumnFamilyStore; import org.apache.cassandra.db.Directories; -import org.apache.cassandra.db.compaction.AbstractCompactedRow; +import org.apache.cassandra.db.rows.UnfilteredRowIterator; import org.apache.cassandra.db.compaction.CompactionTask; import org.apache.cassandra.db.lifecycle.LifecycleTransaction; import org.apache.cassandra.io.sstable.SSTableRewriter; @@ -55,11 +55,11 @@ public abstract class CompactionAwareWriter extends Transactional.AbstractTransa } /** - * Writes a row in an implementation specific way - * @param row the row to append - * @return true if the row was written, false otherwise + * Writes a partition in an implementation specific way + * @param partition the partition to append + * @return true if the partition was written, false otherwise */ - public abstract boolean append(AbstractCompactedRow row); + public abstract boolean append(UnfilteredRowIterator partition); @Override protected Throwable doAbort(Throwable accumulate) @@ -117,4 +117,4 @@ public abstract class CompactionAwareWriter extends Transactional.AbstractTransa return directory; } -} \ No newline at end of file +} diff --git a/src/java/org/apache/cassandra/db/compaction/writers/DefaultCompactionWriter.java b/src/java/org/apache/cassandra/db/compaction/writers/DefaultCompactionWriter.java index 7d88458206..8fc7bec8a5 100644 --- a/src/java/org/apache/cassandra/db/compaction/writers/DefaultCompactionWriter.java +++ b/src/java/org/apache/cassandra/db/compaction/writers/DefaultCompactionWriter.java @@ -25,7 +25,8 @@ import org.slf4j.Logger; import org.slf4j.LoggerFactory; import org.apache.cassandra.db.ColumnFamilyStore; -import org.apache.cassandra.db.compaction.AbstractCompactedRow; +import org.apache.cassandra.db.SerializationHeader; +import org.apache.cassandra.db.rows.UnfilteredRowIterator; import org.apache.cassandra.db.compaction.OperationType; import org.apache.cassandra.db.lifecycle.LifecycleTransaction; import org.apache.cassandra.io.sstable.Descriptor; @@ -54,14 +55,15 @@ public class DefaultCompactionWriter extends CompactionAwareWriter minRepairedAt, cfs.metadata, cfs.partitioner, - new MetadataCollector(txn.originals(), cfs.metadata.comparator, 0)); + new MetadataCollector(txn.originals(), cfs.metadata.comparator, 0), + SerializationHeader.make(cfs.metadata, nonExpiredSSTables)); sstableWriter.switchWriter(writer); } @Override - public boolean append(AbstractCompactedRow row) + public boolean append(UnfilteredRowIterator partition) { - return sstableWriter.append(row) != null; + return sstableWriter.append(partition) != null; } @Override @@ -69,4 +71,4 @@ public class DefaultCompactionWriter extends CompactionAwareWriter { return estimatedTotalKeys; } -} \ No newline at end of file +} diff --git a/src/java/org/apache/cassandra/db/compaction/writers/MajorLeveledCompactionWriter.java b/src/java/org/apache/cassandra/db/compaction/writers/MajorLeveledCompactionWriter.java index 95d7a0c82e..5328fa520d 100644 --- a/src/java/org/apache/cassandra/db/compaction/writers/MajorLeveledCompactionWriter.java +++ b/src/java/org/apache/cassandra/db/compaction/writers/MajorLeveledCompactionWriter.java @@ -25,7 +25,8 @@ import org.slf4j.LoggerFactory; import org.apache.cassandra.db.ColumnFamilyStore; import org.apache.cassandra.db.RowIndexEntry; -import org.apache.cassandra.db.compaction.AbstractCompactedRow; +import org.apache.cassandra.db.SerializationHeader; +import org.apache.cassandra.db.rows.UnfilteredRowIterator; import org.apache.cassandra.db.compaction.LeveledManifest; import org.apache.cassandra.db.compaction.OperationType; import org.apache.cassandra.db.lifecycle.LifecycleTransaction; @@ -68,16 +69,17 @@ public class MajorLeveledCompactionWriter extends CompactionAwareWriter minRepairedAt, cfs.metadata, cfs.partitioner, - new MetadataCollector(allSSTables, cfs.metadata.comparator, currentLevel, skipAncestors)); + new MetadataCollector(allSSTables, cfs.metadata.comparator, currentLevel, skipAncestors), + SerializationHeader.make(cfs.metadata, nonExpiredSSTables)); sstableWriter.switchWriter(writer); } @Override @SuppressWarnings("resource") - public boolean append(AbstractCompactedRow row) + public boolean append(UnfilteredRowIterator partition) { long posBefore = sstableWriter.currentWriter().getOnDiskFilePointer(); - RowIndexEntry rie = sstableWriter.append(row); + RowIndexEntry rie = sstableWriter.append(partition); totalWrittenInLevel += sstableWriter.currentWriter().getOnDiskFilePointer() - posBefore; partitionsWritten++; if (sstableWriter.currentWriter().getOnDiskFilePointer() > maxSSTableSize) @@ -95,7 +97,8 @@ public class MajorLeveledCompactionWriter extends CompactionAwareWriter minRepairedAt, cfs.metadata, cfs.partitioner, - new MetadataCollector(allSSTables, cfs.metadata.comparator, currentLevel, skipAncestors)); + new MetadataCollector(allSSTables, cfs.metadata.comparator, currentLevel, skipAncestors), + SerializationHeader.make(cfs.metadata, nonExpiredSSTables)); sstableWriter.switchWriter(writer); partitionsWritten = 0; sstablesWritten++; @@ -103,4 +106,4 @@ public class MajorLeveledCompactionWriter extends CompactionAwareWriter return rie != null; } -} \ No newline at end of file +} diff --git a/src/java/org/apache/cassandra/db/compaction/writers/MaxSSTableSizeWriter.java b/src/java/org/apache/cassandra/db/compaction/writers/MaxSSTableSizeWriter.java index d30a61208e..4832fd5b82 100644 --- a/src/java/org/apache/cassandra/db/compaction/writers/MaxSSTableSizeWriter.java +++ b/src/java/org/apache/cassandra/db/compaction/writers/MaxSSTableSizeWriter.java @@ -22,7 +22,8 @@ import java.util.Set; import org.apache.cassandra.db.ColumnFamilyStore; import org.apache.cassandra.db.RowIndexEntry; -import org.apache.cassandra.db.compaction.AbstractCompactedRow; +import org.apache.cassandra.db.SerializationHeader; +import org.apache.cassandra.db.rows.UnfilteredRowIterator; import org.apache.cassandra.db.compaction.OperationType; import org.apache.cassandra.db.lifecycle.LifecycleTransaction; import org.apache.cassandra.io.sstable.Descriptor; @@ -57,14 +58,15 @@ public class MaxSSTableSizeWriter extends CompactionAwareWriter minRepairedAt, cfs.metadata, cfs.partitioner, - new MetadataCollector(allSSTables, cfs.metadata.comparator, level)); + new MetadataCollector(allSSTables, cfs.metadata.comparator, level), + SerializationHeader.make(cfs.metadata, nonExpiredSSTables)); sstableWriter.switchWriter(writer); } @Override - public boolean append(AbstractCompactedRow row) + public boolean append(UnfilteredRowIterator partition) { - RowIndexEntry rie = sstableWriter.append(row); + RowIndexEntry rie = sstableWriter.append(partition); if (sstableWriter.currentWriter().getOnDiskFilePointer() > maxSSTableSize) { File sstableDirectory = cfs.directories.getLocationForDisk(getWriteDirectory(expectedWriteSize)); @@ -74,7 +76,8 @@ public class MaxSSTableSizeWriter extends CompactionAwareWriter minRepairedAt, cfs.metadata, cfs.partitioner, - new MetadataCollector(allSSTables, cfs.metadata.comparator, level)); + new MetadataCollector(allSSTables, cfs.metadata.comparator, level), + SerializationHeader.make(cfs.metadata, nonExpiredSSTables)); sstableWriter.switchWriter(writer); } @@ -86,4 +89,4 @@ public class MaxSSTableSizeWriter extends CompactionAwareWriter { return estimatedTotalKeys; } -} \ No newline at end of file +} diff --git a/src/java/org/apache/cassandra/db/compaction/writers/SplittingSizeTieredCompactionWriter.java b/src/java/org/apache/cassandra/db/compaction/writers/SplittingSizeTieredCompactionWriter.java index 9ff1325b49..ba85eeff7b 100644 --- a/src/java/org/apache/cassandra/db/compaction/writers/SplittingSizeTieredCompactionWriter.java +++ b/src/java/org/apache/cassandra/db/compaction/writers/SplittingSizeTieredCompactionWriter.java @@ -26,7 +26,8 @@ import org.slf4j.LoggerFactory; import org.apache.cassandra.db.ColumnFamilyStore; import org.apache.cassandra.db.RowIndexEntry; -import org.apache.cassandra.db.compaction.AbstractCompactedRow; +import org.apache.cassandra.db.SerializationHeader; +import org.apache.cassandra.db.rows.UnfilteredRowIterator; import org.apache.cassandra.db.compaction.OperationType; import org.apache.cassandra.db.lifecycle.LifecycleTransaction; import org.apache.cassandra.io.sstable.Descriptor; @@ -90,16 +91,17 @@ public class SplittingSizeTieredCompactionWriter extends CompactionAwareWriter minRepairedAt, cfs.metadata, cfs.partitioner, - new MetadataCollector(allSSTables, cfs.metadata.comparator, 0)); + new MetadataCollector(allSSTables, cfs.metadata.comparator, 0), + SerializationHeader.make(cfs.metadata, nonExpiredSSTables)); sstableWriter.switchWriter(writer); logger.debug("Ratios={}, expectedKeys = {}, totalSize = {}, currentPartitionsToWrite = {}, currentBytesToWrite = {}", ratios, estimatedTotalKeys, totalSize, currentPartitionsToWrite, currentBytesToWrite); } @Override - public boolean append(AbstractCompactedRow row) + public boolean append(UnfilteredRowIterator partition) { - RowIndexEntry rie = sstableWriter.append(row); + RowIndexEntry rie = sstableWriter.append(partition); if (sstableWriter.currentWriter().getOnDiskFilePointer() > currentBytesToWrite && currentRatioIndex < ratios.length - 1) // if we underestimate how many keys we have, the last sstable might get more than we expect { currentRatioIndex++; @@ -112,10 +114,11 @@ public class SplittingSizeTieredCompactionWriter extends CompactionAwareWriter minRepairedAt, cfs.metadata, cfs.partitioner, - new MetadataCollector(allSSTables, cfs.metadata.comparator, 0)); + new MetadataCollector(allSSTables, cfs.metadata.comparator, 0), + SerializationHeader.make(cfs.metadata, nonExpiredSSTables)); sstableWriter.switchWriter(writer); logger.debug("Switching writer, currentPartitionsToWrite = {}", currentPartitionsToWrite); } return rie != null; } -} \ No newline at end of file +} diff --git a/src/java/org/apache/cassandra/db/composites/AbstractCType.java b/src/java/org/apache/cassandra/db/composites/AbstractCType.java deleted file mode 100644 index a982280e78..0000000000 --- a/src/java/org/apache/cassandra/db/composites/AbstractCType.java +++ /dev/null @@ -1,394 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.apache.cassandra.db.composites; - -import java.io.DataInput; -import java.io.IOException; -import java.nio.ByteBuffer; -import java.util.Comparator; - -import org.apache.cassandra.db.Cell; -import org.apache.cassandra.db.DeletionInfo; -import org.apache.cassandra.db.NativeCell; -import org.apache.cassandra.db.RangeTombstone; -import org.apache.cassandra.db.TypeSizes; -import org.apache.cassandra.db.filter.ColumnSlice; -import org.apache.cassandra.db.filter.SliceQueryFilter; -import org.apache.cassandra.db.marshal.AbstractCompositeType; -import org.apache.cassandra.db.marshal.AbstractType; -import org.apache.cassandra.io.IVersionedSerializer; -import org.apache.cassandra.io.util.DataOutputPlus; -import org.apache.cassandra.utils.ByteBufferUtil; - -import static org.apache.cassandra.io.sstable.IndexHelper.IndexInfo; - -public abstract class AbstractCType implements CType -{ - static final Comparator rightNativeCell = new Comparator() - { - public int compare(Cell o1, Cell o2) - { - return -((NativeCell) o2).compareTo(o1.name()); - } - }; - - static final Comparator neitherNativeCell = new Comparator() - { - public int compare(Cell o1, Cell o2) - { - return compareUnsigned(o1.name(), o2.name()); - } - }; - - // only one or the other of these will ever be used - static final Comparator asymmetricRightNativeCell = new Comparator() - { - public int compare(Object o1, Object o2) - { - return -((NativeCell) o2).compareTo((Composite) o1); - } - }; - - static final Comparator asymmetricNeitherNativeCell = new Comparator() - { - public int compare(Object o1, Object o2) - { - return compareUnsigned((Composite) o1, ((Cell) o2).name()); - } - }; - - private final Comparator reverseComparator; - private final Comparator indexComparator; - private final Comparator indexReverseComparator; - - private final Serializer serializer; - - private final IVersionedSerializer sliceSerializer; - private final IVersionedSerializer sliceQueryFilterSerializer; - private final DeletionInfo.Serializer deletionInfoSerializer; - private final RangeTombstone.Serializer rangeTombstoneSerializer; - - protected final boolean isByteOrderComparable; - - protected AbstractCType(boolean isByteOrderComparable) - { - reverseComparator = new Comparator() - { - public int compare(Composite c1, Composite c2) - { - return AbstractCType.this.compare(c2, c1); - } - }; - indexComparator = new Comparator() - { - public int compare(IndexInfo o1, IndexInfo o2) - { - return AbstractCType.this.compare(o1.lastName, o2.lastName); - } - }; - indexReverseComparator = new Comparator() - { - public int compare(IndexInfo o1, IndexInfo o2) - { - return AbstractCType.this.compare(o1.firstName, o2.firstName); - } - }; - - serializer = new Serializer(this); - - sliceSerializer = new ColumnSlice.Serializer(this); - sliceQueryFilterSerializer = new SliceQueryFilter.Serializer(this); - deletionInfoSerializer = new DeletionInfo.Serializer(this); - rangeTombstoneSerializer = new RangeTombstone.Serializer(this); - this.isByteOrderComparable = isByteOrderComparable; - } - - protected static boolean isByteOrderComparable(Iterable> types) - { - boolean isByteOrderComparable = true; - for (AbstractType type : types) - isByteOrderComparable &= type.isByteOrderComparable(); - return isByteOrderComparable; - } - - static int compareUnsigned(Composite c1, Composite c2) - { - if (c1.isStatic() != c2.isStatic()) - { - // Static sorts before non-static no matter what, except for empty which - // always sort first - if (c1.isEmpty()) - return c2.isEmpty() ? 0 : -1; - if (c2.isEmpty()) - return 1; - return c1.isStatic() ? -1 : 1; - } - - int s1 = c1.size(); - int s2 = c2.size(); - int minSize = Math.min(s1, s2); - - for (int i = 0; i < minSize; i++) - { - int cmp = ByteBufferUtil.compareUnsigned(c1.get(i), c2.get(i)); - if (cmp != 0) - return cmp; - } - - if (s1 == s2) - return c1.eoc().compareTo(c2.eoc()); - return s1 < s2 ? c1.eoc().prefixComparisonResult : -c2.eoc().prefixComparisonResult; - } - - public int compare(Composite c1, Composite c2) - { - if (c1.isStatic() != c2.isStatic()) - { - // Static sorts before non-static no matter what, except for empty which - // always sort first - if (c1.isEmpty()) - return c2.isEmpty() ? 0 : -1; - if (c2.isEmpty()) - return 1; - return c1.isStatic() ? -1 : 1; - } - - int s1 = c1.size(); - int s2 = c2.size(); - int minSize = Math.min(s1, s2); - - for (int i = 0; i < minSize; i++) - { - int cmp = isByteOrderComparable - ? ByteBufferUtil.compareUnsigned(c1.get(i), c2.get(i)) - : subtype(i).compare(c1.get(i), c2.get(i)); - if (cmp != 0) - return cmp; - } - - if (s1 == s2) - return c1.eoc().compareTo(c2.eoc()); - return s1 < s2 ? c1.eoc().prefixComparisonResult : -c2.eoc().prefixComparisonResult; - } - - protected Comparator getByteOrderColumnComparator(boolean isRightNative) - { - if (isRightNative) - return rightNativeCell; - return neitherNativeCell; - } - - protected Comparator getByteOrderAsymmetricColumnComparator(boolean isRightNative) - { - if (isRightNative) - return asymmetricRightNativeCell; - return asymmetricNeitherNativeCell; - } - - public void validate(Composite name) - { - ByteBuffer previous = null; - for (int i = 0; i < name.size(); i++) - { - AbstractType comparator = subtype(i); - ByteBuffer value = name.get(i); - comparator.validateCollectionMember(value, previous); - previous = value; - } - } - - public boolean isCompatibleWith(CType previous) - { - if (this == previous) - return true; - - // Extending with new components is fine, shrinking is not - if (size() < previous.size()) - return false; - - for (int i = 0; i < previous.size(); i++) - { - AbstractType tprev = previous.subtype(i); - AbstractType tnew = subtype(i); - if (!tnew.isCompatibleWith(tprev)) - return false; - } - return true; - } - - public String getString(Composite c) - { - StringBuilder sb = new StringBuilder(); - for (int i = 0; i < c.size(); i++) - { - if (i > 0) - sb.append(":"); - sb.append(AbstractCompositeType.escape(subtype(i).getString(c.get(i)))); - } - switch (c.eoc()) - { - case START: - sb.append(":_"); - break; - case END: - sb.append(":!"); - break; - } - return sb.toString(); - } - - public Composite make(Object... components) - { - if (components.length > size()) - throw new IllegalArgumentException("Too many components, max is " + size()); - - CBuilder builder = builder(); - for (int i = 0; i < components.length; i++) - { - Object obj = components[i]; - if (obj instanceof ByteBuffer) - builder.add((ByteBuffer)obj); - else - builder.add(obj); - } - return builder.build(); - } - - public CType.Serializer serializer() - { - return serializer; - } - - public Comparator reverseComparator() - { - return reverseComparator; - } - - public Comparator indexComparator() - { - return indexComparator; - } - - public Comparator indexReverseComparator() - { - return indexReverseComparator; - } - - public IVersionedSerializer sliceSerializer() - { - return sliceSerializer; - } - - public IVersionedSerializer sliceQueryFilterSerializer() - { - return sliceQueryFilterSerializer; - } - - public DeletionInfo.Serializer deletionInfoSerializer() - { - return deletionInfoSerializer; - } - - public RangeTombstone.Serializer rangeTombstoneSerializer() - { - return rangeTombstoneSerializer; - } - - @Override - public boolean equals(Object o) - { - if (this == o) - return true; - - if (o == null) - return false; - - if (!getClass().equals(o.getClass())) - return false; - - CType c = (CType)o; - if (size() != c.size()) - return false; - - for (int i = 0; i < size(); i++) - { - if (!subtype(i).equals(c.subtype(i))) - return false; - } - return true; - } - - @Override - public int hashCode() - { - int h = 31; - for (int i = 0; i < size(); i++) - h += subtype(i).hashCode(); - return h + getClass().hashCode(); - } - - @Override - public String toString() - { - return asAbstractType().toString(); - } - - protected static ByteBuffer sliceBytes(ByteBuffer bb, int offs, int length) - { - ByteBuffer copy = bb.duplicate(); - copy.position(offs); - copy.limit(offs + length); - return copy; - } - - protected static void checkRemaining(ByteBuffer bb, int offs, int length) - { - if (offs + length > bb.limit()) - throw new IllegalArgumentException("Not enough bytes"); - } - - private static class Serializer implements CType.Serializer - { - private final CType type; - - public Serializer(CType type) - { - this.type = type; - } - - public void serialize(Composite c, DataOutputPlus out) throws IOException - { - ByteBufferUtil.writeWithShortLength(c.toByteBuffer(), out); - } - - public Composite deserialize(DataInput in) throws IOException - { - return type.fromByteBuffer(ByteBufferUtil.readWithShortLength(in)); - } - - public long serializedSize(Composite c, TypeSizes type) - { - return type.sizeofWithShortLength(c.toByteBuffer()); - } - - public void skip(DataInput in) throws IOException - { - ByteBufferUtil.skipShortLength(in); - } - } -} diff --git a/src/java/org/apache/cassandra/db/composites/AbstractCellNameType.java b/src/java/org/apache/cassandra/db/composites/AbstractCellNameType.java deleted file mode 100644 index c62f890ccb..0000000000 --- a/src/java/org/apache/cassandra/db/composites/AbstractCellNameType.java +++ /dev/null @@ -1,454 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.apache.cassandra.db.composites; - -import java.io.DataInput; -import java.io.IOException; -import java.nio.ByteBuffer; -import java.util.*; - -import com.google.common.collect.AbstractIterator; -import org.apache.cassandra.config.CFMetaData; -import org.apache.cassandra.config.ColumnDefinition; -import org.apache.cassandra.cql3.CQL3Row; -import org.apache.cassandra.cql3.ColumnIdentifier; -import org.apache.cassandra.db.*; -import org.apache.cassandra.db.filter.IDiskAtomFilter; -import org.apache.cassandra.db.filter.NamesQueryFilter; -import org.apache.cassandra.db.marshal.AbstractType; -import org.apache.cassandra.db.marshal.CollectionType; -import org.apache.cassandra.db.marshal.ColumnToCollectionType; -import org.apache.cassandra.io.ISerializer; -import org.apache.cassandra.io.IVersionedSerializer; -import org.apache.cassandra.io.util.DataOutputPlus; -import org.apache.cassandra.utils.ByteBufferUtil; - -public abstract class AbstractCellNameType extends AbstractCType implements CellNameType -{ - final Comparator columnComparator; - private final Comparator columnReverseComparator; - final Comparator asymmetricComparator; - private final Comparator onDiskAtomComparator; - - private final ISerializer cellSerializer; - private final ColumnSerializer columnSerializer; - private final OnDiskAtom.Serializer onDiskAtomSerializer; - private final IVersionedSerializer namesQueryFilterSerializer; - private final IVersionedSerializer diskAtomFilterSerializer; - - protected AbstractCellNameType(boolean isByteOrderComparable) - { - super(isByteOrderComparable); - columnComparator = new Comparator() - { - public int compare(Cell c1, Cell c2) - { - return AbstractCellNameType.this.compare(c1.name(), c2.name()); - } - }; - asymmetricComparator = new Comparator() - { - public int compare(Object c1, Object c2) - { - return AbstractCellNameType.this.compare((Composite) c1, ((Cell) c2).name()); - } - }; - columnReverseComparator = new Comparator() - { - public int compare(Cell c1, Cell c2) - { - return AbstractCellNameType.this.compare(c2.name(), c1.name()); - } - }; - onDiskAtomComparator = new Comparator() - { - public int compare(OnDiskAtom c1, OnDiskAtom c2) - { - int comp = AbstractCellNameType.this.compare(c1.name(), c2.name()); - if (comp != 0) - return comp; - - if (c1 instanceof RangeTombstone) - { - if (c2 instanceof RangeTombstone) - { - RangeTombstone t1 = (RangeTombstone)c1; - RangeTombstone t2 = (RangeTombstone)c2; - int comp2 = AbstractCellNameType.this.compare(t1.max, t2.max); - return comp2 == 0 ? t1.data.compareTo(t2.data) : comp2; - } - else - { - return -1; - } - } - else - { - return c2 instanceof RangeTombstone ? 1 : 0; - } - } - }; - - // A trivial wrapped over the composite serializer - cellSerializer = new ISerializer() - { - public void serialize(CellName c, DataOutputPlus out) throws IOException - { - serializer().serialize(c, out); - } - - public CellName deserialize(DataInput in) throws IOException - { - Composite ct = serializer().deserialize(in); - if (ct.isEmpty()) - throw ColumnSerializer.CorruptColumnException.create(in, ByteBufferUtil.EMPTY_BYTE_BUFFER); - - assert ct instanceof CellName : ct; - return (CellName)ct; - } - - public long serializedSize(CellName c, TypeSizes type) - { - return serializer().serializedSize(c, type); - } - }; - columnSerializer = new ColumnSerializer(this); - onDiskAtomSerializer = new OnDiskAtom.Serializer(this); - namesQueryFilterSerializer = new NamesQueryFilter.Serializer(this); - diskAtomFilterSerializer = new IDiskAtomFilter.Serializer(this); - } - - public final Comparator columnComparator(boolean isRightNative) - { - if (!isByteOrderComparable) - return columnComparator; - return getByteOrderColumnComparator(isRightNative); - } - - public final Comparator asymmetricColumnComparator(boolean isRightNative) - { - if (!isByteOrderComparable) - return asymmetricComparator; - return getByteOrderAsymmetricColumnComparator(isRightNative); - } - - public Comparator columnReverseComparator() - { - return columnReverseComparator; - } - - public Comparator onDiskAtomComparator() - { - return onDiskAtomComparator; - } - - public ISerializer cellSerializer() - { - return cellSerializer; - } - - public ColumnSerializer columnSerializer() - { - return columnSerializer; - } - - public OnDiskAtom.Serializer onDiskAtomSerializer() - { - return onDiskAtomSerializer; - } - - public IVersionedSerializer namesQueryFilterSerializer() - { - return namesQueryFilterSerializer; - } - - public IVersionedSerializer diskAtomFilterSerializer() - { - return diskAtomFilterSerializer; - } - - public CellName cellFromByteBuffer(ByteBuffer bytes) - { - // we're not guaranteed to get a CellName back from fromByteBuffer(), so it's on the caller to guarantee this - return (CellName)fromByteBuffer(bytes); - } - - public CellName create(Composite prefix, ColumnDefinition column, ByteBuffer collectionElement) - { - throw new UnsupportedOperationException(); - } - - public CellName rowMarker(Composite prefix) - { - throw new UnsupportedOperationException(); - } - - public Composite staticPrefix() - { - throw new UnsupportedOperationException(); - } - - public boolean hasCollections() - { - return false; - } - - public boolean supportCollections() - { - return false; - } - - public ColumnToCollectionType collectionType() - { - throw new UnsupportedOperationException(); - } - - public CellNameType addOrUpdateCollection(ColumnIdentifier columnName, CollectionType newCollection) - { - throw new UnsupportedOperationException(); - } - - @Override - public Composite make(Object... components) - { - return components.length == size() ? makeCellName(components) : super.make(components); - } - - public CellName makeCellName(Object... components) - { - ByteBuffer[] rawComponents = new ByteBuffer[components.length]; - for (int i = 0; i < components.length; i++) - { - Object c = components[i]; - if (c instanceof ByteBuffer) - { - rawComponents[i] = (ByteBuffer)c; - } - else - { - AbstractType type = subtype(i); - // If it's a collection type, we need to find the right collection and use the key comparator (since we're building a cell name) - if (type instanceof ColumnToCollectionType) - { - assert i > 0; - type = ((ColumnToCollectionType)type).defined.get(rawComponents[i-1]).nameComparator(); - } - rawComponents[i] = ((AbstractType)type).decompose(c); - } - } - return makeCellName(rawComponents); - } - - protected abstract CellName makeCellName(ByteBuffer[] components); - - protected static CQL3Row.Builder makeDenseCQL3RowBuilder(final long now) - { - return new CQL3Row.Builder() - { - public CQL3Row.RowIterator group(Iterator cells) - { - return new DenseRowIterator(cells, now); - } - }; - } - - private static class DenseRowIterator extends AbstractIterator implements CQL3Row.RowIterator - { - private final Iterator cells; - private final long now; - - public DenseRowIterator(Iterator cells, long now) - { - this.cells = cells; - this.now = now; - } - - public CQL3Row getStaticRow() - { - // There can't be static columns in dense tables - return null; - } - - protected CQL3Row computeNext() - { - while (cells.hasNext()) - { - final Cell cell = cells.next(); - if (!cell.isLive(now)) - continue; - - return new CQL3Row() - { - public ByteBuffer getClusteringColumn(int i) - { - return cell.name().get(i); - } - - public Cell getColumn(ColumnIdentifier name) - { - return cell; - } - - public List getMultiCellColumn(ColumnIdentifier name) - { - return null; - } - }; - } - return endOfData(); - } - } - - protected static CQL3Row.Builder makeSparseCQL3RowBuilder(final CFMetaData cfMetaData, final CellNameType type, final long now) - { - return new CQL3Row.Builder() - { - public CQL3Row.RowIterator group(Iterator cells) - { - return new SparseRowIterator(cfMetaData, type, cells, now); - } - }; - } - - private static class SparseRowIterator extends AbstractIterator implements CQL3Row.RowIterator - { - private final CFMetaData cfMetaData; - private final CellNameType type; - private final Iterator cells; - private final long now; - private final CQL3Row staticRow; - - private Cell nextCell; - private CellName previous; - private CQL3RowOfSparse currentRow; - - public SparseRowIterator(CFMetaData cfMetaData, CellNameType type, Iterator cells, long now) - { - this.cfMetaData = cfMetaData; - this.type = type; - this.cells = cells; - this.now = now; - this.staticRow = hasNextCell() && nextCell.name().isStatic() - ? computeNext() - : null; - } - - public CQL3Row getStaticRow() - { - return staticRow; - } - - private boolean hasNextCell() - { - if (nextCell != null) - return true; - - while (cells.hasNext()) - { - Cell cell = cells.next(); - if (!cell.isLive(now)) - continue; - - nextCell = cell; - return true; - } - return false; - } - - protected CQL3Row computeNext() - { - while (hasNextCell()) - { - CQL3Row toReturn = null; - CellName current = nextCell.name(); - if (currentRow == null || !current.isSameCQL3RowAs(type, previous)) - { - toReturn = currentRow; - currentRow = new CQL3RowOfSparse(cfMetaData, current); - } - currentRow.add(nextCell); - nextCell = null; - previous = current; - - if (toReturn != null) - return toReturn; - } - if (currentRow != null) - { - CQL3Row toReturn = currentRow; - currentRow = null; - return toReturn; - } - return endOfData(); - } - } - - private static class CQL3RowOfSparse implements CQL3Row - { - private final CFMetaData cfMetaData; - private final CellName cell; - private Map columns; - private Map> collections; - - CQL3RowOfSparse(CFMetaData metadata, CellName cell) - { - this.cfMetaData = metadata; - this.cell = cell; - } - - public ByteBuffer getClusteringColumn(int i) - { - return cell.get(i); - } - - void add(Cell cell) - { - CellName cellName = cell.name(); - ColumnIdentifier columnName = cellName.cql3ColumnName(cfMetaData); - if (cellName.isCollectionCell()) - { - if (collections == null) - collections = new HashMap<>(); - - List values = collections.get(columnName); - if (values == null) - { - values = new ArrayList(); - collections.put(columnName, values); - } - values.add(cell); - } - else - { - if (columns == null) - columns = new HashMap<>(); - columns.put(columnName, cell); - } - } - - public Cell getColumn(ColumnIdentifier name) - { - return columns == null ? null : columns.get(name); - } - - public List getMultiCellColumn(ColumnIdentifier name) - { - return collections == null ? null : collections.get(name); - } - } -} diff --git a/src/java/org/apache/cassandra/db/composites/AbstractComposite.java b/src/java/org/apache/cassandra/db/composites/AbstractComposite.java deleted file mode 100644 index 14fa16c86e..0000000000 --- a/src/java/org/apache/cassandra/db/composites/AbstractComposite.java +++ /dev/null @@ -1,141 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.apache.cassandra.db.composites; - -import java.nio.ByteBuffer; - -import org.apache.cassandra.db.filter.ColumnSlice; -import org.apache.cassandra.db.marshal.CompositeType; -import org.apache.cassandra.utils.ByteBufferUtil; - -public abstract class AbstractComposite implements Composite -{ - public boolean isEmpty() - { - return size() == 0; - } - - public boolean isStatic() - { - return false; - } - - public EOC eoc() - { - return EOC.NONE; - } - - public Composite start() - { - return withEOC(EOC.START); - } - - public Composite end() - { - return withEOC(EOC.END); - } - - public Composite withEOC(EOC newEoc) - { - // Note: CompositeBound overwrite this so we assume the EOC of this is NONE - switch (newEoc) - { - case START: - return BoundedComposite.startOf(this); - case END: - return BoundedComposite.endOf(this); - default: - return this; - } - } - - public ColumnSlice slice() - { - return new ColumnSlice(start(), end()); - } - - public ByteBuffer toByteBuffer() - { - // This is the legacy format of composites. - // See org.apache.cassandra.db.marshal.CompositeType for details. - ByteBuffer result = ByteBuffer.allocate(dataSize() + 3 * size() + (isStatic() ? 2 : 0)); - if (isStatic()) - ByteBufferUtil.writeShortLength(result, CompositeType.STATIC_MARKER); - - for (int i = 0; i < size(); i++) - { - ByteBuffer bb = get(i); - ByteBufferUtil.writeShortLength(result, bb.remaining()); - result.put(bb.duplicate()); - result.put((byte)0); - } - result.flip(); - return result; - } - - public int dataSize() - { - int size = 0; - for (int i = 0; i < size(); i++) - size += get(i).remaining(); - return size; - } - - public boolean isPrefixOf(CType type, Composite c) - { - if (size() > c.size() || isStatic() != c.isStatic()) - return false; - - for (int i = 0; i < size(); i++) - { - if (type.subtype(i).compare(get(i), c.get(i)) != 0) - return false; - } - return true; - } - - @Override - public boolean equals(Object o) - { - if (this == o) - return true; - - if(!(o instanceof Composite)) - return false; - - Composite c = (Composite)o; - if (size() != c.size() || isStatic() != c.isStatic()) - return false; - - for (int i = 0; i < size(); i++) - { - if (!get(i).equals(c.get(i))) - return false; - } - return eoc() == c.eoc(); - } - - @Override - public int hashCode() - { - int h = 31; - for (int i = 0; i < size(); i++) - h += get(i).hashCode(); - return h + eoc().hashCode() + (isStatic() ? 1 : 0); - } -} diff --git a/src/java/org/apache/cassandra/db/composites/AbstractCompoundCellNameType.java b/src/java/org/apache/cassandra/db/composites/AbstractCompoundCellNameType.java deleted file mode 100644 index bf303a784e..0000000000 --- a/src/java/org/apache/cassandra/db/composites/AbstractCompoundCellNameType.java +++ /dev/null @@ -1,295 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.apache.cassandra.db.composites; - -import java.io.DataInput; -import java.io.IOException; -import java.nio.ByteBuffer; - -import org.apache.cassandra.db.marshal.AbstractType; -import org.apache.cassandra.db.marshal.CompositeType; - -public abstract class AbstractCompoundCellNameType extends AbstractCellNameType -{ - protected final CompoundCType clusteringType; - protected final CompoundCType fullType; - - protected final int clusteringSize; - protected final int fullSize; - - protected AbstractCompoundCellNameType(CompoundCType clusteringType, CompoundCType fullType) - { - super(isByteOrderComparable(fullType.types)); - this.clusteringType = clusteringType; - this.fullType = fullType; - - this.clusteringSize = clusteringType.size(); - this.fullSize = fullType.size(); - } - - public int clusteringPrefixSize() - { - return clusteringSize; - } - - public boolean isCompound() - { - return true; - } - - public int size() - { - return fullSize; - } - - public AbstractType subtype(int i) - { - return fullType.subtype(i); - } - - public CBuilder prefixBuilder() - { - return clusteringType.builder(); - } - - public CBuilder builder() - { - return new CompoundCType.CompoundCBuilder(this); - } - - @Override - public Composite fromByteBuffer(ByteBuffer bytes) - { - if (!bytes.hasRemaining()) - return Composites.EMPTY; - - ByteBuffer[] elements = new ByteBuffer[fullSize]; - int idx = bytes.position(), i = 0; - byte eoc = 0; - - boolean isStatic = false; - if (CompositeType.isStaticName(bytes)) - { - isStatic = true; - idx += 2; - } - - while (idx < bytes.limit()) - { - checkRemaining(bytes, idx, 2); - int length = bytes.getShort(idx) & 0xFFFF; - idx += 2; - - checkRemaining(bytes, idx, length + 1); - elements[i++] = sliceBytes(bytes, idx, length); - idx += length; - eoc = bytes.get(idx++); - } - - return makeWith(elements, i, Composite.EOC.from(eoc), isStatic); - } - - public AbstractType asAbstractType() - { - return CompositeType.getInstance(fullType.types); - } - - public Deserializer newDeserializer(DataInput in) - { - return new CompositeDeserializer(this, in); - } - - protected CellName makeCellName(ByteBuffer[] components) - { - return (CellName)makeWith(components, components.length, Composite.EOC.NONE, false); - } - - protected abstract Composite makeWith(ByteBuffer[] components, int size, Composite.EOC eoc, boolean isStatic); - protected abstract Composite copyAndMakeWith(ByteBuffer[] components, int size, Composite.EOC eoc, boolean isStatic); - - private static class CompositeDeserializer implements CellNameType.Deserializer - { - private static byte[] EMPTY = new byte[0]; - - private final AbstractCompoundCellNameType type; - private final DataInput in; - - private byte[] nextFull; - private int nextIdx; - - private final ByteBuffer[] nextComponents; - private int nextSize; - private Composite.EOC nextEOC; - private boolean nextIsStatic; - - public CompositeDeserializer(AbstractCompoundCellNameType type, DataInput in) - { - this.type = type; - this.in = in; - this.nextComponents = new ByteBuffer[type.size()]; - } - - public boolean hasNext() throws IOException - { - if (nextFull == null) - maybeReadNext(); - return nextFull != EMPTY; - } - - public boolean hasUnprocessed() throws IOException - { - return nextFull != null; - } - - public int compareNextTo(Composite composite) throws IOException - { - maybeReadNext(); - - if (composite.isEmpty()) - return nextFull == EMPTY ? 0 : 1; - - if (nextFull == EMPTY) - return -1; - - if (nextIsStatic != composite.isStatic()) - return nextIsStatic ? -1 : 1; - - ByteBuffer previous = null; - for (int i = 0; i < composite.size(); i++) - { - if (!hasComponent(i)) - return nextEOC == Composite.EOC.END ? 1 : -1; - - AbstractType comparator = type.subtype(i); - ByteBuffer value1 = nextComponents[i]; - ByteBuffer value2 = composite.get(i); - - int cmp = comparator.compareCollectionMembers(value1, value2, previous); - if (cmp != 0) - return cmp; - - previous = value1; - } - - // If we have more component than composite - if (!allComponentsDeserialized() || composite.size() < nextSize) - return composite.eoc() == Composite.EOC.END ? -1 : 1; - - // same size, check eoc - if (nextEOC != composite.eoc()) - { - switch (nextEOC) - { - case START: return -1; - case END: return 1; - case NONE: return composite.eoc() == Composite.EOC.START ? 1 : -1; - } - } - - return 0; - } - - private boolean hasComponent(int i) - { - while (i >= nextSize && deserializeOne()) - continue; - - return i < nextSize; - } - - private int readShort() - { - return ((nextFull[nextIdx++] & 0xFF) << 8) | (nextFull[nextIdx++] & 0xFF); - } - - private int peekShort() - { - return ((nextFull[nextIdx] & 0xFF) << 8) | (nextFull[nextIdx+1] & 0xFF); - } - - private boolean deserializeOne() - { - if (allComponentsDeserialized()) - return false; - - int length = readShort(); - ByteBuffer component = ByteBuffer.wrap(nextFull, nextIdx, length); - nextIdx += length; - nextComponents[nextSize++] = component; - nextEOC = Composite.EOC.from(nextFull[nextIdx++]); - return true; - } - - private void deserializeAll() - { - while (deserializeOne()) - continue; - } - - private boolean allComponentsDeserialized() - { - return nextIdx >= nextFull.length; - } - - private void maybeReadNext() throws IOException - { - if (nextFull != null) - return; - - nextIdx = 0; - nextSize = 0; - - int length = in.readShort() & 0xFFFF; - // Note that empty is ok because it marks the end of row - if (length == 0) - { - nextFull = EMPTY; - return; - } - - nextFull = new byte[length]; - in.readFully(nextFull); - - // Is is a static? - nextIsStatic = false; - if (peekShort() == CompositeType.STATIC_MARKER) - { - nextIsStatic = true; - readShort(); // Skip the static marker - } - } - - public Composite readNext() throws IOException - { - maybeReadNext(); - if (nextFull == EMPTY) - return Composites.EMPTY; - - deserializeAll(); - Composite c = type.copyAndMakeWith(nextComponents, nextSize, nextEOC, nextIsStatic); - nextFull = null; - return c; - } - - public void skipNext() throws IOException - { - maybeReadNext(); - nextFull = null; - } - } -} diff --git a/src/java/org/apache/cassandra/db/composites/AbstractSimpleCellNameType.java b/src/java/org/apache/cassandra/db/composites/AbstractSimpleCellNameType.java deleted file mode 100644 index b3f477835a..0000000000 --- a/src/java/org/apache/cassandra/db/composites/AbstractSimpleCellNameType.java +++ /dev/null @@ -1,210 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.apache.cassandra.db.composites; - -import java.io.DataInput; -import java.io.IOException; -import java.nio.ByteBuffer; -import java.util.Comparator; - -import net.nicoulaj.compilecommand.annotations.Inline; -import org.apache.cassandra.db.Cell; -import org.apache.cassandra.db.NativeCell; -import org.apache.cassandra.db.marshal.AbstractType; -import org.apache.cassandra.utils.ByteBufferUtil; - -public abstract class AbstractSimpleCellNameType extends AbstractCellNameType -{ - protected final AbstractType type; - - static final Comparator rightNativeCell = new Comparator() - { - public int compare(Cell o1, Cell o2) - { - return -((NativeCell) o2).compareToSimple(o1.name()); - } - }; - - static final Comparator neitherNativeCell = new Comparator() - { - public int compare(Cell o1, Cell o2) - { - return compareUnsigned(o1.name(), o2.name()); - } - }; - - // only one or the other of these will ever be used - static final Comparator asymmetricRightNativeCell = new Comparator() - { - public int compare(Object o1, Object o2) - { - return -((NativeCell) o2).compareToSimple((Composite) o1); - } - }; - - static final Comparator asymmetricNeitherNativeCell = new Comparator() - { - public int compare(Object o1, Object o2) - { - return compareUnsigned((Composite) o1, ((Cell) o2).name()); - } - }; - - protected AbstractSimpleCellNameType(AbstractType type) - { - super(type.isByteOrderComparable()); - this.type = type; - } - - public boolean isCompound() - { - return false; - } - - public int size() - { - return 1; - } - - @Inline - static int compareUnsigned(Composite c1, Composite c2) - { - ByteBuffer b1 = c1.toByteBuffer(); - ByteBuffer b2 = c2.toByteBuffer(); - return ByteBufferUtil.compareUnsigned(b1, b2); - } - - public int compare(Composite c1, Composite c2) - { - if (isByteOrderComparable) - return compareUnsigned(c1, c2); - - assert !(c1.isEmpty() | c2.isEmpty()); - return type.compare(c1.get(0), c2.get(0)); - } - - protected Comparator getByteOrderColumnComparator(boolean isRightNative) - { - if (isRightNative) - return rightNativeCell; - return neitherNativeCell; - } - - protected Comparator getByteOrderAsymmetricColumnComparator(boolean isRightNative) - { - if (isRightNative) - return asymmetricRightNativeCell; - return asymmetricNeitherNativeCell; - } - - public AbstractType subtype(int i) - { - if (i != 0) - throw new IllegalArgumentException(); - return type; - } - - protected CellName makeCellName(ByteBuffer[] components) - { - assert components.length == 1; - return cellFromByteBuffer(components[0]); - } - - public CBuilder builder() - { - return new SimpleCType.SimpleCBuilder(this); - } - - public AbstractType asAbstractType() - { - return type; - } - - public Deserializer newDeserializer(DataInput in) - { - return new SimpleDeserializer(this, in); - } - - private static class SimpleDeserializer implements CellNameType.Deserializer - { - private final AbstractSimpleCellNameType type; - private ByteBuffer next; - private final DataInput in; - - public SimpleDeserializer(AbstractSimpleCellNameType type, DataInput in) - { - this.type = type; - this.in = in; - } - - public boolean hasNext() throws IOException - { - if (next == null) - maybeReadNext(); - - return next.hasRemaining(); - } - - public boolean hasUnprocessed() throws IOException - { - return next != null; - } - - public int compareNextTo(Composite composite) throws IOException - { - maybeReadNext(); - - if (composite.isEmpty()) - return next.hasRemaining() ? 1 : 0; - - return type.subtype(0).compare(next, composite.get(0)); - } - - private void maybeReadNext() throws IOException - { - if (next != null) - return; - - int length = in.readShort() & 0xFFFF; - // Note that empty is ok because it marks the end of row - if (length == 0) - { - next = ByteBufferUtil.EMPTY_BYTE_BUFFER; - return; - } - - byte[] b = new byte[length]; - in.readFully(b); - next = ByteBuffer.wrap(b); - } - - public Composite readNext() throws IOException - { - maybeReadNext(); - Composite c = type.fromByteBuffer(next); - next = null; - return c; - } - - public void skipNext() throws IOException - { - maybeReadNext(); - next = null; - } - } -} diff --git a/src/java/org/apache/cassandra/db/composites/BoundedComposite.java b/src/java/org/apache/cassandra/db/composites/BoundedComposite.java deleted file mode 100644 index 7f596feaad..0000000000 --- a/src/java/org/apache/cassandra/db/composites/BoundedComposite.java +++ /dev/null @@ -1,104 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.apache.cassandra.db.composites; - -import java.nio.ByteBuffer; - -import org.apache.cassandra.config.CFMetaData; -import org.apache.cassandra.utils.memory.AbstractAllocator; -import org.apache.cassandra.utils.ObjectSizes; - -/** - * Wraps another Composite and adds an EOC byte to track whether this is a slice start or end. - */ -public class BoundedComposite extends AbstractComposite -{ - private static final long EMPTY_SIZE = ObjectSizes.measure(new BoundedComposite(null, false)); - - private final Composite wrapped; - private final boolean isStart; - - private BoundedComposite(Composite wrapped, boolean isStart) - { - this.wrapped = wrapped; - this.isStart = isStart; - } - - static Composite startOf(Composite c) - { - return new BoundedComposite(c, true); - } - - static Composite endOf(Composite c) - { - return new BoundedComposite(c, false); - } - - public int size() - { - return wrapped.size(); - } - - public boolean isStatic() - { - return wrapped.isStatic(); - } - - public ByteBuffer get(int i) - { - return wrapped.get(i); - } - - @Override - public EOC eoc() - { - return isStart ? EOC.START : EOC.END; - } - - @Override - public Composite withEOC(EOC eoc) - { - switch (eoc) - { - case START: - return isStart ? this : startOf(wrapped); - case END: - return isStart ? endOf(wrapped) : this; - default: - return wrapped; - } - } - - @Override - public ByteBuffer toByteBuffer() - { - ByteBuffer bb = wrapped.toByteBuffer(); - bb.put(bb.remaining() - 1, (byte)(isStart ? -1 : 1)); - return bb; - } - - public long unsharedHeapSize() - { - return EMPTY_SIZE + wrapped.unsharedHeapSize(); - } - - public Composite copy(CFMetaData cfm, AbstractAllocator allocator) - { - return new BoundedComposite(wrapped.copy(cfm, allocator), isStart); - } -} diff --git a/src/java/org/apache/cassandra/db/composites/CType.java b/src/java/org/apache/cassandra/db/composites/CType.java deleted file mode 100644 index 7f703138c8..0000000000 --- a/src/java/org/apache/cassandra/db/composites/CType.java +++ /dev/null @@ -1,141 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.apache.cassandra.db.composites; - -import java.io.DataInput; -import java.io.IOException; -import java.nio.ByteBuffer; -import java.util.Comparator; - -import org.apache.cassandra.db.DeletionInfo; -import org.apache.cassandra.db.RangeTombstone; -import org.apache.cassandra.db.filter.ColumnSlice; -import org.apache.cassandra.db.filter.SliceQueryFilter; -import org.apache.cassandra.db.marshal.AbstractType; -import org.apache.cassandra.io.ISerializer; -import org.apache.cassandra.io.IVersionedSerializer; - -import static org.apache.cassandra.io.sstable.IndexHelper.IndexInfo; - -/** - * A type for a Composite. - * - * There is essentially 2 types of Composite and such of CType: - * 1. the "simple" ones, see SimpleCType. - * 2. the "truly-composite" ones, see CompositeCType. - * - * API-wise, a CType is simply a collection of AbstractType with a few utility - * methods. - */ -public interface CType extends Comparator -{ - /** - * Returns whether this is a "truly-composite" underneath. - */ - public boolean isCompound(); - - /** - * The number of subtypes for this CType. - */ - public int size(); - - int compare(Composite o1, Composite o2); - - /** - * Gets a subtype of this CType. - */ - public AbstractType subtype(int i); - - /** - * A builder of Composite. - */ - public CBuilder builder(); - - /** - * Convenience method to build composites from their component. - * - * The arguments can be either ByteBuffer or actual objects of the type - * corresponding to their position. - */ - public Composite make(Object... components); - - /** - * Validates a composite. - */ - public void validate(Composite name); - - /** - * Converts a composite to a user-readable string. - */ - public String getString(Composite c); - - /** - * See AbstractType#isCompatibleWith. - */ - public boolean isCompatibleWith(CType previous); - - /** - * Returns a new CType that is equivalent to this CType but with - * one of the subtype replaced by the provided new type. - */ - public CType setSubtype(int position, AbstractType newType); - - /** - * Deserialize a Composite from a ByteBuffer. - * - * This is meant for thrift to convert the fully serialized buffer we - * get from the clients to composites. - */ - public Composite fromByteBuffer(ByteBuffer bb); - - /** - * Returns a AbstractType corresponding to this CType for thrift sake. - * - * If the CType is a "simple" one, this just return the wrapped type, otherwise - * it returns the corresponding org.apache.cassandra.db.marshal.CompositeType. - * - * This is only meant to be use for backward compatibility (particularly for - * thrift) but it's not meant to be used internally. - */ - public AbstractType asAbstractType(); - - - /**********************************************************/ - - /* - * Follows a number of per-CType instances for the Comparator and Serializer used throughout - * the code. The reason we need this is that we want the per-CType/per-CellNameType Composite/CellName - * serializers, which means the following instances have to depend on the type too. - */ - - public Comparator reverseComparator(); - public Comparator indexComparator(); - public Comparator indexReverseComparator(); - - public Serializer serializer(); - - public IVersionedSerializer sliceSerializer(); - public IVersionedSerializer sliceQueryFilterSerializer(); - public DeletionInfo.Serializer deletionInfoSerializer(); - public RangeTombstone.Serializer rangeTombstoneSerializer(); - - public interface Serializer extends ISerializer - { - public void skip(DataInput in) throws IOException; - } -} diff --git a/src/java/org/apache/cassandra/db/composites/CellName.java b/src/java/org/apache/cassandra/db/composites/CellName.java deleted file mode 100644 index 4d778d3715..0000000000 --- a/src/java/org/apache/cassandra/db/composites/CellName.java +++ /dev/null @@ -1,78 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.apache.cassandra.db.composites; - -import java.nio.ByteBuffer; - -import org.apache.cassandra.config.CFMetaData; -import org.apache.cassandra.cql3.ColumnIdentifier; -import org.apache.cassandra.utils.memory.AbstractAllocator; - -/** - * A CellName is a Composite, but for which, for the sake of CQL3, we - * distinguish different parts: a CellName has first a number of clustering - * components, followed by the CQL3 column name, and then possibly followed by - * a collection element part. - * - * The clustering prefix can itself be composed of multiple component. It can - * also be empty if the table has no clustering keys. In general, the CQL3 - * column name follows. However, some type of COMPACT STORAGE layout do not - * store the CQL3 column name in the cell name and so this part can be null (we - * call "dense" the cells whose name don't store the CQL3 column name). - * - * Lastly, if the cell is part of a CQL3 collection, we'll have a last - * component (a UUID for lists, an element for sets and a key for maps). - */ -public interface CellName extends Composite -{ - /** - * The number of clustering components. - * - * It can be 0 if the table has no clustering columns, and it can be - * equal to size() if the table is dense() (in which case cql3ColumnName() - * will be null). - */ - public int clusteringSize(); - - /** - * The name of the CQL3 column this cell represents. - * - * Will be null for cells of "dense" tables. - * @param metadata - */ - public ColumnIdentifier cql3ColumnName(CFMetaData metadata); - - /** - * The value of the collection element, or null if the cell is not part - * of a collection (i.e. if !isCollectionCell()). - */ - public ByteBuffer collectionElement(); - public boolean isCollectionCell(); - - /** - * Whether this cell is part of the same CQL3 row as the other cell. - */ - public boolean isSameCQL3RowAs(CellNameType type, CellName other); - - // If cellnames were sharing some prefix components, this will break it, so - // we might want to try to do better. - @Override - public CellName copy(CFMetaData cfm, AbstractAllocator allocator); - - public long unsharedHeapSizeExcludingData(); -} diff --git a/src/java/org/apache/cassandra/db/composites/CellNameType.java b/src/java/org/apache/cassandra/db/composites/CellNameType.java deleted file mode 100644 index 6c896601fc..0000000000 --- a/src/java/org/apache/cassandra/db/composites/CellNameType.java +++ /dev/null @@ -1,215 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.apache.cassandra.db.composites; - -import java.io.DataInput; -import java.io.IOException; -import java.nio.ByteBuffer; -import java.util.Comparator; - -import org.apache.cassandra.config.CFMetaData; -import org.apache.cassandra.config.ColumnDefinition; -import org.apache.cassandra.cql3.CQL3Row; -import org.apache.cassandra.cql3.ColumnIdentifier; -import org.apache.cassandra.db.Cell; -import org.apache.cassandra.db.ColumnSerializer; -import org.apache.cassandra.db.OnDiskAtom; -import org.apache.cassandra.db.filter.IDiskAtomFilter; -import org.apache.cassandra.db.filter.NamesQueryFilter; -import org.apache.cassandra.db.marshal.AbstractType; -import org.apache.cassandra.db.marshal.CollectionType; -import org.apache.cassandra.db.marshal.ColumnToCollectionType; -import org.apache.cassandra.io.ISerializer; -import org.apache.cassandra.io.IVersionedSerializer; - -/** - * The type of CellNames. - * - * In the same way that a CellName is a Composite, a CellNameType is a CType, but - * with a number of method specific to cell names. - * - * On top of the dichotomy simple/truly-composite of composites, cell names comes - * in 2 variants: "dense" and "sparse". The sparse ones are CellName where one of - * the component (the last or second-to-last for collections) is used to store the - * CQL3 column name. Dense are those for which it's not the case. - * - * In other words, we have 4 types of CellName/CellNameType which correspond to the - * 4 type of table layout that we need to distinguish: - * 1. Simple (non-truly-composite) dense: this is the dynamic thrift CFs whose - * comparator is not composite. - * 2. Composite dense: this is the dynamic thrift CFs with a CompositeType comparator. - * 3. Simple (non-truly-composite) sparse: this is the thrift static CFs (that - * don't have a composite comparator). - * 4. Composite sparse: this is the CQL3 layout (note that this is the only one that - * support collections). - */ -public interface CellNameType extends CType -{ - /** - * Whether or not the cell names for this type are dense. - */ - public boolean isDense(); - - /** - * The number of clustering columns for the table this is the type of. - */ - public int clusteringPrefixSize(); - - /** - * A builder for the clustering prefix. - */ - public CBuilder prefixBuilder(); - - /** - * The prefix to use for static columns. - * - * Note that create() methods below for creating CellName automatically handle static columns already - * for convenience, and so there is not need to pass this prefix for them. There is few other cases - * where we need the prefix directly however. - */ - public Composite staticPrefix(); - - /** - * Whether or not there is some collections defined in this type. - */ - public boolean hasCollections(); - - /** - * Whether or not this type layout support collections. - */ - public boolean supportCollections(); - - /** - * The type of the collections (or null if the type does not have any non-frozen collections). - */ - public ColumnToCollectionType collectionType(); - - /** - * Return the new type obtained by adding/updating to the new collection type for the provided column name - * to this type. - */ - public CellNameType addOrUpdateCollection(ColumnIdentifier columnName, CollectionType newCollection); - - /** - * Returns a new CellNameType that is equivalent to this one but with one - * of the subtype replaced by the provided new type. - */ - @Override - public CellNameType setSubtype(int position, AbstractType newType); - - /** - * Creates a row marker for the CQL3 having the provided clustering prefix. - * - * Note that this is only valid for CQL3 tables (isCompound() and !isDense()) and should - * only be called for them. - */ - public CellName rowMarker(Composite prefix); - - /** - * Creates a new CellName given a clustering prefix and a CQL3 column. - * - * Note that for dense types, the column can be null as a shortcut for designing the only - * COMPACT_VALUE column of the table. - */ - public CellName create(Composite prefix, ColumnDefinition column); - - /** - * Creates a new collection CellName given a clustering prefix, a CQL3 column and the collection element. - */ - public CellName create(Composite prefix, ColumnDefinition column, ByteBuffer collectionElement); - - /** - * Convenience method to create cell names given its components. - * - * This is equivalent to CType#make() but return a full cell name (and thus - * require all the components of the name). - */ - public CellName makeCellName(Object... components); - - /** - * Deserialize a Composite from a ByteBuffer. - * - * This is equilvalent to CType#fromByteBuffer but assumes the buffer is a full cell - * name. This is meant for thrift to convert the fully serialized buffer we - * get from the clients. - */ - public CellName cellFromByteBuffer(ByteBuffer bb); - - /** - * Creates a new CQL3Row builder for this type. See CQL3Row for details. - */ - public CQL3Row.Builder CQL3RowBuilder(CFMetaData metadata, long now); - - // The two following methods are used to pass the declared regular column names (in CFMetaData) - // to the CellNameType. This is only used for optimization sake, see SparseCellNameType. - public void addCQL3Column(ColumnIdentifier id); - public void removeCQL3Column(ColumnIdentifier id); - - /** - * Creates a new Deserializer. This is used by AtomDeserializer to do incremental and on-demand - * deserialization of the on disk atoms. See AtomDeserializer for details. - */ - public Deserializer newDeserializer(DataInput in); - - /* - * Same as in CType, follows a number of per-CellNameType instances for the Comparator and Serializer used - * throughout the code (those that require full CellName versus just Composite). - */ - - // Ultimately, those might be split into an IVersionedSerializer and an ISSTableSerializer - public ISerializer cellSerializer(); - - public Comparator columnComparator(boolean isRightNative); - public Comparator asymmetricColumnComparator(boolean isRightNative); - public Comparator columnReverseComparator(); - public Comparator onDiskAtomComparator(); - - public ColumnSerializer columnSerializer(); - public OnDiskAtom.Serializer onDiskAtomSerializer(); - public IVersionedSerializer namesQueryFilterSerializer(); - public IVersionedSerializer diskAtomFilterSerializer(); - - public interface Deserializer - { - /** - * Whether this deserializer is done or not, i.e. whether we're reached the end of row marker. - */ - public boolean hasNext() throws IOException; - - /** - * Whether or not some name has been read but not consumed by readNext. - */ - public boolean hasUnprocessed() throws IOException; - - /** - * Compare the next name to read to the provided Composite. - * This does not consume the next name. - */ - public int compareNextTo(Composite composite) throws IOException; - - /** - * Actually consume the next name and return it. - */ - public Composite readNext() throws IOException; - - /** - * Skip the next name (consuming it). - */ - public void skipNext() throws IOException; - } -} diff --git a/src/java/org/apache/cassandra/db/composites/CellNames.java b/src/java/org/apache/cassandra/db/composites/CellNames.java deleted file mode 100644 index f73f7a7e5f..0000000000 --- a/src/java/org/apache/cassandra/db/composites/CellNames.java +++ /dev/null @@ -1,109 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.apache.cassandra.db.composites; - -import java.nio.ByteBuffer; -import java.util.List; - -import org.apache.cassandra.cql3.ColumnIdentifier; -import org.apache.cassandra.db.Cell; -import org.apache.cassandra.db.marshal.AbstractType; -import org.apache.cassandra.db.marshal.ColumnToCollectionType; -import org.apache.cassandra.db.marshal.CompositeType; -import org.apache.cassandra.db.marshal.UTF8Type; - -public abstract class CellNames -{ - private CellNames() {} - - public static CellNameType fromAbstractType(AbstractType type, boolean isDense) - { - if (isDense) - { - if (type instanceof CompositeType) - { - return new CompoundDenseCellNameType(((CompositeType)type).types); - } - else - { - return new SimpleDenseCellNameType(type); - } - } - else - { - if (type instanceof CompositeType) - { - List> types = ((CompositeType)type).types; - if (types.get(types.size() - 1) instanceof ColumnToCollectionType) - { - // We don't allow collection for super columns, so the "name" type *must* be UTF8 - assert types.get(types.size() - 2) instanceof UTF8Type; - return new CompoundSparseCellNameType.WithCollection(types.subList(0, types.size() - 2), (ColumnToCollectionType)types.get(types.size() - 1)); - } - else - { - AbstractType nameType = types.get(types.size() - 1); - return new CompoundSparseCellNameType(types.subList(0, types.size() - 1), nameType); - } - } - else - { - assert type != null; - return new SimpleSparseCellNameType(type); - } - } - } - - // Mainly for tests and a few cases where we know what we need and didn't wanted to pass the type around. - // Avoid in general, prefer the CellNameType methods. - public static CellName simpleDense(ByteBuffer bb) - { - assert bb.hasRemaining(); - return new SimpleDenseCellName(bb); - } - - public static CellName simpleSparse(ColumnIdentifier identifier) - { - return new SimpleSparseCellName(identifier); - } - - // Mainly for tests and a few cases where we know what we need and didn't wanted to pass the type around - // Avoid in general, prefer the CellNameType methods. - public static CellName compositeDense(ByteBuffer... bbs) - { - return new CompoundDenseCellName(bbs); - } - - public static CellName compositeSparse(ByteBuffer[] bbs, ColumnIdentifier identifier, boolean isStatic) - { - return new CompoundSparseCellName(bbs, identifier, isStatic); - } - - public static CellName compositeSparseWithCollection(ByteBuffer[] bbs, ByteBuffer collectionElement, ColumnIdentifier identifier, boolean isStatic) - { - return new CompoundSparseCellName.WithCollection(bbs, identifier, collectionElement, isStatic); - } - - public static String getColumnsString(CellNameType type, Iterable columns) - { - StringBuilder builder = new StringBuilder(); - for (Cell cell : columns) - builder.append(cell.getString(type)).append(","); - return builder.toString(); - } -} diff --git a/src/java/org/apache/cassandra/db/composites/Composite.java b/src/java/org/apache/cassandra/db/composites/Composite.java deleted file mode 100644 index b15daeffde..0000000000 --- a/src/java/org/apache/cassandra/db/composites/Composite.java +++ /dev/null @@ -1,78 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.apache.cassandra.db.composites; - -import java.nio.ByteBuffer; - -import org.apache.cassandra.cache.IMeasurableMemory; -import org.apache.cassandra.config.CFMetaData; -import org.apache.cassandra.db.filter.ColumnSlice; -import org.apache.cassandra.utils.memory.AbstractAllocator; - -/** - * A composite value. - * - * This can be though as a list of ByteBuffer, except that this also include an - * 'end-of-component' flag, that allow precise selection of composite ranges. - * - * We also make a difference between "true" composites and the "simple" ones. The - * non-truly composite will have a size() == 1 but differs from true composites with - * size() == 1 in the way they are stored. Most code shouldn't have to care about the - * difference. - */ -public interface Composite extends IMeasurableMemory -{ - public enum EOC - { - START(-1), NONE(-1), END(1); - - // If composite p has this EOC and is a strict prefix of composite c, then this - // the result of the comparison of p and c. Basically, p sorts before c unless - // it's EOC is END. - public final int prefixComparisonResult; - - private EOC(int prefixComparisonResult) - { - this.prefixComparisonResult = prefixComparisonResult; - } - - public static EOC from(int eoc) - { - return eoc == 0 ? NONE : (eoc < 0 ? START : END); - } - } - - public int size(); - public boolean isEmpty(); - public ByteBuffer get(int i); - - public EOC eoc(); - public Composite withEOC(EOC eoc); - public Composite start(); - public Composite end(); - public ColumnSlice slice(); - - public boolean isStatic(); - - public boolean isPrefixOf(CType type, Composite other); - - public ByteBuffer toByteBuffer(); - - public int dataSize(); - public Composite copy(CFMetaData cfm, AbstractAllocator allocator); -} diff --git a/src/java/org/apache/cassandra/db/composites/Composites.java b/src/java/org/apache/cassandra/db/composites/Composites.java deleted file mode 100644 index fa0df48c52..0000000000 --- a/src/java/org/apache/cassandra/db/composites/Composites.java +++ /dev/null @@ -1,150 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.apache.cassandra.db.composites; - -import java.nio.ByteBuffer; -import java.util.List; - -import org.apache.cassandra.config.CFMetaData; -import org.apache.cassandra.db.filter.ColumnSlice; -import org.apache.cassandra.utils.ByteBufferUtil; -import org.apache.cassandra.utils.memory.AbstractAllocator; - -import com.google.common.base.Function; -import com.google.common.collect.Lists; - -public abstract class Composites -{ - private Composites() {} - - public static final Composite EMPTY = new EmptyComposite(); - - /** - * Converts the specified Composites into ByteBuffers. - * - * @param composites the composites to convert. - * @return the ByteBuffers corresponding to the specified Composites. - */ - public static List toByteBuffers(List composites) - { - return Lists.transform(composites, new Function() - { - public ByteBuffer apply(Composite composite) - { - return composite.toByteBuffer(); - } - }); - } - - static final CBuilder EMPTY_BUILDER = new CBuilder() - { - public int remainingCount() { return 0; } - - public CBuilder add(ByteBuffer value) { throw new IllegalStateException(); } - public CBuilder add(Object value) { throw new IllegalStateException(); } - - public Composite build() { return EMPTY; } - public Composite buildWith(ByteBuffer value) { throw new IllegalStateException(); } - public Composite buildWith(List values) { throw new IllegalStateException(); } - }; - - private static class EmptyComposite implements Composite - { - public boolean isEmpty() - { - return true; - } - - public int size() - { - return 0; - } - - public ByteBuffer get(int i) - { - if (i > 0) - throw new IndexOutOfBoundsException(); - - return ByteBufferUtil.EMPTY_BYTE_BUFFER; - } - - public EOC eoc() - { - return EOC.NONE; - } - - public Composite start() - { - // Note that SimpleCType/AbstractSimpleCellNameType compare method - // indirectly rely on the fact that EMPTY == EMPTY.start() == EMPTY.end() - // (or more precisely on the fact that the EOC is NONE for all of those). - return this; - } - - public Composite end() - { - // Note that SimpleCType/AbstractSimpleCellNameType compare method - // indirectly rely on the fact that EMPTY == EMPTY.start() == EMPTY.end() - // (or more precisely on the fact that the EOC is NONE for all of those). - return this; - } - - public Composite withEOC(EOC newEoc) - { - // Note that SimpleCType/AbstractSimpleCellNameType compare method - // indirectly rely on the fact that EMPTY == EMPTY.start() == EMPTY.end() - // (or more precisely on the fact that the EOC is NONE for all of those). - return this; - } - - public ColumnSlice slice() - { - return ColumnSlice.ALL_COLUMNS; - } - - public ByteBuffer toByteBuffer() - { - return ByteBufferUtil.EMPTY_BYTE_BUFFER; - } - - public boolean isStatic() - { - return false; - } - - public int dataSize() - { - return 0; - } - - public long unsharedHeapSize() - { - return 0; - } - - public boolean isPrefixOf(CType type, Composite c) - { - return true; - } - - public Composite copy(CFMetaData cfm, AbstractAllocator allocator) - { - return this; - } - } -} diff --git a/src/java/org/apache/cassandra/db/composites/CompositesBuilder.java b/src/java/org/apache/cassandra/db/composites/CompositesBuilder.java deleted file mode 100644 index 25a510fb39..0000000000 --- a/src/java/org/apache/cassandra/db/composites/CompositesBuilder.java +++ /dev/null @@ -1,313 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.apache.cassandra.db.composites; - -import java.nio.ByteBuffer; -import java.util.*; - -import org.apache.cassandra.db.composites.Composite.EOC; -import org.apache.cassandra.utils.ByteBufferUtil; - -import static java.util.Collections.singletonList; - -/** - * Builder that allow to build multiple composites at the same time. - */ -public final class CompositesBuilder -{ - /** - * The composite type. - */ - private final CType ctype; - - /** - * The elements of the composites - */ - private final List> elementsList = new ArrayList<>(); - - /** - * The number of elements that have been added. - */ - private int size; - - /** - * true if the composites have been build, false otherwise. - */ - private boolean built; - - /** - * true if the composites contains some null elements. - */ - private boolean containsNull; - - /** - * true if some empty collection have been added. - */ - private boolean hasMissingElements; - - /** - * true if the composites contains some unset elements. - */ - private boolean containsUnset; - - public CompositesBuilder(CType ctype) - { - this.ctype = ctype; - } - - /** - * Adds the specified element to all the composites. - *

- * If this builder contains 2 composites: A-B and A-C a call to this method to add D will result in the composites: - * A-B-D and A-C-D. - *

- * - * @param value the value of the next element - * @return this CompositeBuilder - */ - public CompositesBuilder addElementToAll(ByteBuffer value) - { - checkUpdateable(); - - if (isEmpty()) - elementsList.add(new ArrayList()); - - for (int i = 0, m = elementsList.size(); i < m; i++) - { - if (value == null) - containsNull = true; - if (value == ByteBufferUtil.UNSET_BYTE_BUFFER) - containsUnset = true; - elementsList.get(i).add(value); - } - size++; - return this; - } - - /** - * Adds individually each of the specified elements to the end of all of the existing composites. - *

- * If this builder contains 2 composites: A-B and A-C a call to this method to add D and E will result in the 4 - * composites: A-B-D, A-B-E, A-C-D and A-C-E. - *

- * - * @param values the elements to add - * @return this CompositeBuilder - */ - public CompositesBuilder addEachElementToAll(List values) - { - checkUpdateable(); - - if (isEmpty()) - elementsList.add(new ArrayList()); - - if (values.isEmpty()) - { - hasMissingElements = true; - } - else - { - for (int i = 0, m = elementsList.size(); i < m; i++) - { - List oldComposite = elementsList.remove(0); - - for (int j = 0, n = values.size(); j < n; j++) - { - List newComposite = new ArrayList<>(oldComposite); - elementsList.add(newComposite); - - ByteBuffer value = values.get(j); - - if (value == null) - containsNull = true; - if (value == ByteBufferUtil.UNSET_BYTE_BUFFER) - containsUnset = true; - - newComposite.add(values.get(j)); - } - } - } - size++; - return this; - } - - - /** - * Adds individually each of the specified list of elements to the end of all of the existing composites. - *

- * If this builder contains 2 composites: A-B and A-C a call to this method to add [[D, E], [F, G]] will result in the 4 - * composites: A-B-D-E, A-B-F-G, A-C-D-E and A-C-F-G. - *

- * - * @param values the elements to add - * @return this CompositeBuilder - */ - public CompositesBuilder addAllElementsToAll(List> values) - { - checkUpdateable(); - - if (isEmpty()) - elementsList.add(new ArrayList()); - - if (values.isEmpty()) - { - hasMissingElements = true; - } - else - { - for (int i = 0, m = elementsList.size(); i < m; i++) - { - List oldComposite = elementsList.remove(0); - - for (int j = 0, n = values.size(); j < n; j++) - { - List newComposite = new ArrayList<>(oldComposite); - elementsList.add(newComposite); - - List value = values.get(j); - - if (value.isEmpty()) - hasMissingElements = true; - - if (value.contains(null)) - containsNull = true; - if (value.contains(ByteBufferUtil.UNSET_BYTE_BUFFER)) - containsUnset = true; - - newComposite.addAll(value); - } - } - size += values.get(0).size(); - } - return this; - } - - /** - * Returns the number of elements that can be added to the composites. - * - * @return the number of elements that can be added to the composites. - */ - public int remainingCount() - { - return ctype.size() - size; - } - - /** - * Checks if some elements can still be added to the composites. - * - * @return true if it is possible to add more elements to the composites, false otherwise. - */ - public boolean hasRemaining() - { - return remainingCount() > 0; - } - - /** - * Checks if this builder is empty. - * - * @return true if this builder is empty, false otherwise. - */ - public boolean isEmpty() - { - return elementsList.isEmpty(); - } - - /** - * Checks if the composites contains null elements. - * - * @return true if the composites contains null elements, false otherwise. - */ - public boolean containsNull() - { - return containsNull; - } - - /** - * Checks if some empty list of values have been added - * @return true if the composites have some missing elements, false otherwise. - */ - public boolean hasMissingElements() - { - return hasMissingElements; - } - - /** - * Checks if the composites contains unset elements. - * - * @return true if the composites contains unset elements, false otherwise. - */ - public boolean containsUnset() - { - return containsUnset; - } - - /** - * Builds the Composites. - * - * @return the composites - */ - public List build() - { - return buildWithEOC(EOC.NONE); - } - - /** - * Builds the Composites with the specified EOC. - * - * @return the composites - */ - public List buildWithEOC(EOC eoc) - { - built = true; - - if (hasMissingElements) - return Collections.emptyList(); - - CBuilder builder = ctype.builder(); - - if (elementsList.isEmpty()) - return singletonList(builder.build().withEOC(eoc)); - - // Use a Set to sort if needed and eliminate duplicates - Set set = newSet(); - - for (int i = 0, m = elementsList.size(); i < m; i++) - { - List elements = elementsList.get(i); - set.add(builder.buildWith(elements).withEOC(eoc)); - } - - return new ArrayList<>(set); - } - - /** - * Returns a new Set instance that will be used to eliminate duplicates and sort the results. - * - * @return a new Set instance. - */ - private Set newSet() - { - return new TreeSet<>(ctype); - } - - private void checkUpdateable() - { - if (!hasRemaining() || built) - throw new IllegalStateException("this CompositesBuilder cannot be updated anymore"); - } -} diff --git a/src/java/org/apache/cassandra/db/composites/CompoundCType.java b/src/java/org/apache/cassandra/db/composites/CompoundCType.java deleted file mode 100644 index 04587484fb..0000000000 --- a/src/java/org/apache/cassandra/db/composites/CompoundCType.java +++ /dev/null @@ -1,180 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.apache.cassandra.db.composites; - -import java.nio.ByteBuffer; -import java.util.ArrayList; -import java.util.Arrays; -import java.util.List; - -import org.apache.cassandra.db.marshal.AbstractType; -import org.apache.cassandra.db.marshal.CompositeType; - -/** - * A truly-composite CType. - */ -public class CompoundCType extends AbstractCType -{ - final List> types; - - // It's up to the caller to pass a list that is effectively immutable - public CompoundCType(List> types) - { - super(isByteOrderComparable(types)); - this.types = types; - } - - public boolean isCompound() - { - return true; - } - - public int size() - { - return types.size(); - } - - public AbstractType subtype(int i) - { - return types.get(i); - } - - public Composite fromByteBuffer(ByteBuffer bytes) - { - if (!bytes.hasRemaining()) - return Composites.EMPTY; - - ByteBuffer[] elements = new ByteBuffer[size()]; - int idx = bytes.position(), i = 0; - byte eoc = 0; - - boolean isStatic = false; - if (CompositeType.isStaticName(bytes)) - { - isStatic = true; - idx += 2; - } - - while (idx < bytes.limit()) - { - checkRemaining(bytes, idx, 2); - int length = bytes.getShort(idx) & 0xFFFF; - idx += 2; - - checkRemaining(bytes, idx, length + 1); - elements[i++] = sliceBytes(bytes, idx, length); - idx += length; - eoc = bytes.get(idx++); - } - return new CompoundComposite(elements, i, isStatic).withEOC(Composite.EOC.from(eoc)); - } - - public CBuilder builder() - { - return new CompoundCBuilder(this); - } - - public CompoundCType setSubtype(int position, AbstractType newType) - { - List> newTypes = new ArrayList>(types); - newTypes.set(position, newType); - return new CompoundCType(newTypes); - } - - public AbstractType asAbstractType() - { - return CompositeType.getInstance(types); - } - - public static class CompoundCBuilder implements CBuilder - { - private final CType type; - private final ByteBuffer[] values; - private int size; - private boolean built; - - public CompoundCBuilder(CType type) - { - this.type = type; - this.values = new ByteBuffer[type.size()]; - } - - public int remainingCount() - { - return values.length - size; - } - - public CBuilder add(ByteBuffer value) - { - if (isDone()) - throw new IllegalStateException(); - values[size++] = value; - return this; - } - - public CBuilder add(Object value) - { - return add(((AbstractType)type.subtype(size)).decompose(value)); - } - - private boolean isDone() - { - return remainingCount() == 0 || built; - } - - public Composite build() - { - if (size == 0) - return Composites.EMPTY; - - // We don't allow to add more element to a builder that has been built so - // that we don't have to copy values. - built = true; - - // If the builder is full and we're building a dense cell name, then we can - // directly allocate the CellName object as it's complete. - if (size == values.length && type instanceof CellNameType && ((CellNameType)type).isDense()) - return new CompoundDenseCellName(values); - return new CompoundComposite(values, size, false); - } - - public Composite buildWith(ByteBuffer value) - { - ByteBuffer[] newValues = Arrays.copyOf(values, values.length); - newValues[size] = value; - // Same as above - if (size+1 == newValues.length && type instanceof CellNameType && ((CellNameType)type).isDense()) - return new CompoundDenseCellName(newValues); - - return new CompoundComposite(newValues, size+1, false); - } - - public Composite buildWith(List newValues) - { - ByteBuffer[] buffers = Arrays.copyOf(values, values.length); - int newSize = size; - for (ByteBuffer value : newValues) - buffers[newSize++] = value; - - if (newSize == buffers.length && type instanceof CellNameType && ((CellNameType)type).isDense()) - return new CompoundDenseCellName(buffers); - - return new CompoundComposite(buffers, newSize, false); - } - } -} diff --git a/src/java/org/apache/cassandra/db/composites/CompoundComposite.java b/src/java/org/apache/cassandra/db/composites/CompoundComposite.java deleted file mode 100644 index 7a21b01fe8..0000000000 --- a/src/java/org/apache/cassandra/db/composites/CompoundComposite.java +++ /dev/null @@ -1,88 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.apache.cassandra.db.composites; - -import java.nio.ByteBuffer; - -import org.apache.cassandra.config.CFMetaData; -import org.apache.cassandra.utils.ObjectSizes; -import org.apache.cassandra.utils.memory.AbstractAllocator; - -/** - * A "truly-composite" Composite. - */ -public class CompoundComposite extends AbstractComposite -{ - private static final long HEAP_SIZE = ObjectSizes.measure(new CompoundComposite(null, 0, false)); - - // We could use a List, but we'll create such object *a lot* and using a array+size is not - // all that harder, so we save the List object allocation. - final ByteBuffer[] elements; - final int size; - final boolean isStatic; - - CompoundComposite(ByteBuffer[] elements, int size, boolean isStatic) - { - this.elements = elements; - this.size = size; - this.isStatic = isStatic; - } - - public int size() - { - return size; - } - - public ByteBuffer get(int i) - { - // Note: most consumer should validate that i is within bounds. However, for backward compatibility - // reasons, composite dense tables can have names that don't have all their component of the clustering - // columns, which may end up here with i > size(). For those calls, it's actually simpler to return null - // than to force the caller to special case. - return i >= size() ? null : elements[i]; - } - - @Override - public boolean isStatic() - { - return isStatic; - } - - protected ByteBuffer[] elementsCopy(AbstractAllocator allocator) - { - ByteBuffer[] elementsCopy = new ByteBuffer[size]; - for (int i = 0; i < size; i++) - elementsCopy[i] = allocator.clone(elements[i]); - return elementsCopy; - } - - public long unsharedHeapSize() - { - return HEAP_SIZE + ObjectSizes.sizeOnHeapOf(elements); - } - - public long unsharedHeapSizeExcludingData() - { - return HEAP_SIZE + ObjectSizes.sizeOnHeapExcludingData(elements); - } - - public Composite copy(CFMetaData cfm, AbstractAllocator allocator) - { - return new CompoundComposite(elementsCopy(allocator), size, isStatic); - } -} diff --git a/src/java/org/apache/cassandra/db/composites/CompoundDenseCellName.java b/src/java/org/apache/cassandra/db/composites/CompoundDenseCellName.java deleted file mode 100644 index 1f471a8894..0000000000 --- a/src/java/org/apache/cassandra/db/composites/CompoundDenseCellName.java +++ /dev/null @@ -1,86 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.apache.cassandra.db.composites; - -import java.nio.ByteBuffer; - -import org.apache.cassandra.config.CFMetaData; -import org.apache.cassandra.cql3.ColumnIdentifier; -import org.apache.cassandra.utils.memory.AbstractAllocator; -import org.apache.cassandra.utils.ObjectSizes; - -public class CompoundDenseCellName extends CompoundComposite implements CellName -{ - - private static final long HEAP_SIZE = ObjectSizes.measure(new CompoundDenseCellName(new ByteBuffer[0])); - - // Not meant to be used directly, you should use the CellNameType method instead - CompoundDenseCellName(ByteBuffer[] elements) - { - super(elements, elements.length, false); - } - - CompoundDenseCellName(ByteBuffer[] elements, int size) - { - super(elements, size, false); - } - - public int clusteringSize() - { - return size; - } - - public ColumnIdentifier cql3ColumnName(CFMetaData metadata) - { - return null; - } - - public ByteBuffer collectionElement() - { - return null; - } - - public boolean isCollectionCell() - { - return false; - } - - public boolean isSameCQL3RowAs(CellNameType type, CellName other) - { - // Dense cell imply one cell by CQL row so no other cell will be the same row. - return type.compare(this, other) == 0; - } - - @Override - public long unsharedHeapSize() - { - return HEAP_SIZE + ObjectSizes.sizeOnHeapOf(elements); - } - - @Override - public long unsharedHeapSizeExcludingData() - { - return HEAP_SIZE + ObjectSizes.sizeOnHeapExcludingData(elements); - } - - public CellName copy(CFMetaData cfm, AbstractAllocator allocator) - { - return new CompoundDenseCellName(elementsCopy(allocator)); - } - -} diff --git a/src/java/org/apache/cassandra/db/composites/CompoundDenseCellNameType.java b/src/java/org/apache/cassandra/db/composites/CompoundDenseCellNameType.java deleted file mode 100644 index 2e409fbf72..0000000000 --- a/src/java/org/apache/cassandra/db/composites/CompoundDenseCellNameType.java +++ /dev/null @@ -1,87 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.apache.cassandra.db.composites; - -import java.nio.ByteBuffer; -import java.util.Arrays; -import java.util.List; - -import org.apache.cassandra.config.CFMetaData; -import org.apache.cassandra.config.ColumnDefinition; -import org.apache.cassandra.cql3.CQL3Row; -import org.apache.cassandra.cql3.ColumnIdentifier; -import org.apache.cassandra.db.marshal.AbstractType; - -public class CompoundDenseCellNameType extends AbstractCompoundCellNameType -{ - public CompoundDenseCellNameType(List> types) - { - this(new CompoundCType(types)); - } - - private CompoundDenseCellNameType(CompoundCType type) - { - super(type, type); - } - - public CellNameType setSubtype(int position, AbstractType newType) - { - if (position != 0) - throw new IllegalArgumentException(); - return new SimpleDenseCellNameType(newType); - } - - public boolean isDense() - { - return true; - } - - public CellName create(Composite prefix, ColumnDefinition column) - { - // We ignore the column because it's just the COMPACT_VALUE name which is not store in the cell name (and it can be null anyway) - if (prefix instanceof CellName) - return (CellName)prefix; - - // as noted below in makeWith(), compound dense cell names don't have to include all components - assert prefix instanceof CompoundComposite; - CompoundComposite lc = (CompoundComposite)prefix; - return new CompoundDenseCellName(lc.elements, lc.size); - } - - protected Composite makeWith(ByteBuffer[] components, int size, Composite.EOC eoc, boolean isStatic) - { - assert !isStatic; - // A composite dense table cell name don't have to have all the component set to qualify as a - // proper CellName (for backward compatibility reasons mostly), so always return a cellName - CompoundDenseCellName c = new CompoundDenseCellName(components, size); - return eoc != Composite.EOC.NONE ? c.withEOC(eoc) : c; - } - - protected Composite copyAndMakeWith(ByteBuffer[] components, int size, Composite.EOC eoc, boolean isStatic) - { - return makeWith(Arrays.copyOfRange(components, 0, size), size, eoc, isStatic); - } - - public void addCQL3Column(ColumnIdentifier id) {} - public void removeCQL3Column(ColumnIdentifier id) {} - - public CQL3Row.Builder CQL3RowBuilder(CFMetaData metadata, long now) - { - return makeDenseCQL3RowBuilder(now); - } -} diff --git a/src/java/org/apache/cassandra/db/composites/CompoundSparseCellName.java b/src/java/org/apache/cassandra/db/composites/CompoundSparseCellName.java deleted file mode 100644 index 03af6d0190..0000000000 --- a/src/java/org/apache/cassandra/db/composites/CompoundSparseCellName.java +++ /dev/null @@ -1,182 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.apache.cassandra.db.composites; - -import java.nio.ByteBuffer; - -import org.apache.cassandra.config.CFMetaData; -import org.apache.cassandra.cql3.ColumnIdentifier; -import org.apache.cassandra.utils.ByteBufferUtil; -import org.apache.cassandra.utils.memory.AbstractAllocator; -import org.apache.cassandra.utils.ObjectSizes; - -public class CompoundSparseCellName extends CompoundComposite implements CellName -{ - private static final ByteBuffer[] EMPTY_PREFIX = new ByteBuffer[0]; - - private static final long HEAP_SIZE = ObjectSizes.measure(new CompoundSparseCellName(null, false)); - - protected final ColumnIdentifier columnName; - - // Not meant to be used directly, you should use the CellNameType method instead - CompoundSparseCellName(ColumnIdentifier columnName, boolean isStatic) - { - this(EMPTY_PREFIX, columnName, isStatic); - } - - CompoundSparseCellName(ByteBuffer[] elements, ColumnIdentifier columnName, boolean isStatic) - { - this(elements, elements.length, columnName, isStatic); - } - - CompoundSparseCellName(ByteBuffer[] elements, int size, ColumnIdentifier columnName, boolean isStatic) - { - super(elements, size, isStatic); - this.columnName = columnName; - } - - @Override - public long unsharedHeapSize() - { - return HEAP_SIZE + ObjectSizes.sizeOnHeapOf(elements); - } - - @Override - public long unsharedHeapSizeExcludingData() - { - return HEAP_SIZE + ObjectSizes.sizeOnHeapExcludingData(elements); - } - - public int size() - { - return size + 1; - } - - public ByteBuffer get(int i) - { - return i == size ? columnName.bytes : elements[i]; - } - - public int clusteringSize() - { - return size; - } - - public ColumnIdentifier cql3ColumnName(CFMetaData metadata) - { - return columnName; - } - - public ByteBuffer collectionElement() - { - return null; - } - - public boolean isCollectionCell() - { - return false; - } - - public boolean isSameCQL3RowAs(CellNameType type, CellName other) - { - if (clusteringSize() != other.clusteringSize() || other.isStatic() != isStatic()) - return false; - - for (int i = 0; i < clusteringSize(); i++) - { - if (type.subtype(i).compare(elements[i], other.get(i)) != 0) - return false; - } - return true; - } - - public CellName copy(CFMetaData cfm, AbstractAllocator allocator) - { - if (elements.length == 0) - return this; - - // We don't copy columnName because it's interned in SparseCellNameType - return new CompoundSparseCellName(elementsCopy(allocator), columnName, isStatic()); - } - - public static class WithCollection extends CompoundSparseCellName - { - private static final long HEAP_SIZE = ObjectSizes.measure(new WithCollection(null, ByteBufferUtil.EMPTY_BYTE_BUFFER, false)); - - private final ByteBuffer collectionElement; - - WithCollection(ColumnIdentifier columnName, ByteBuffer collectionElement, boolean isStatic) - { - this(EMPTY_PREFIX, columnName, collectionElement, isStatic); - } - - WithCollection(ByteBuffer[] elements, ColumnIdentifier columnName, ByteBuffer collectionElement, boolean isStatic) - { - this(elements, elements.length, columnName, collectionElement, isStatic); - } - - WithCollection(ByteBuffer[] elements, int size, ColumnIdentifier columnName, ByteBuffer collectionElement, boolean isStatic) - { - super(elements, size, columnName, isStatic); - this.collectionElement = collectionElement; - } - - public int size() - { - return size + 2; - } - - public ByteBuffer get(int i) - { - return i == size + 1 ? collectionElement : super.get(i); - } - - @Override - public ByteBuffer collectionElement() - { - return collectionElement; - } - - @Override - public boolean isCollectionCell() - { - return true; - } - - @Override - public CellName copy(CFMetaData cfm, AbstractAllocator allocator) - { - // We don't copy columnName because it's interned in SparseCellNameType - return new CompoundSparseCellName.WithCollection(elements.length == 0 ? elements : elementsCopy(allocator), size, columnName, allocator.clone(collectionElement), isStatic()); - } - - @Override - public long unsharedHeapSize() - { - return HEAP_SIZE + ObjectSizes.sizeOnHeapOf(elements) - + ObjectSizes.sizeOnHeapExcludingData(collectionElement); - } - - @Override - public long unsharedHeapSizeExcludingData() - { - return HEAP_SIZE + ObjectSizes.sizeOnHeapExcludingData(elements) - + ObjectSizes.sizeOnHeapExcludingData(collectionElement); - } - } -} \ No newline at end of file diff --git a/src/java/org/apache/cassandra/db/composites/CompoundSparseCellNameType.java b/src/java/org/apache/cassandra/db/composites/CompoundSparseCellNameType.java deleted file mode 100644 index c88c6f442d..0000000000 --- a/src/java/org/apache/cassandra/db/composites/CompoundSparseCellNameType.java +++ /dev/null @@ -1,330 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.apache.cassandra.db.composites; - -import java.nio.ByteBuffer; -import java.util.*; - -import org.apache.cassandra.config.CFMetaData; -import org.apache.cassandra.config.ColumnDefinition; -import org.apache.cassandra.cql3.CQL3Row; -import org.apache.cassandra.cql3.ColumnIdentifier; -import org.apache.cassandra.db.marshal.*; -import org.apache.cassandra.utils.ByteBufferUtil; -import org.apache.cassandra.utils.memory.AbstractAllocator; - -public class CompoundSparseCellNameType extends AbstractCompoundCellNameType -{ - public static final ColumnIdentifier rowMarkerId = new ColumnIdentifier(ByteBufferUtil.EMPTY_BYTE_BUFFER, UTF8Type.instance); - private static final CellName rowMarkerNoPrefix = new CompoundSparseCellName(rowMarkerId, false); - - // For CQL3 columns, this is always UTF8Type. However, for compatibility with super columns, we need to allow it to be non-UTF8. - private final AbstractType columnNameType; - protected final Map internedIds; - - private final Composite staticPrefix; - - public CompoundSparseCellNameType(List> types) - { - this(types, UTF8Type.instance); - } - - public CompoundSparseCellNameType(List> types, AbstractType columnNameType) - { - this(new CompoundCType(types), columnNameType); - } - - private CompoundSparseCellNameType(CompoundCType clusteringType, AbstractType columnNameType) - { - this(clusteringType, columnNameType, makeCType(clusteringType, columnNameType, null), new HashMap()); - } - - private CompoundSparseCellNameType(CompoundCType clusteringType, AbstractType columnNameType, CompoundCType fullType, Map internedIds) - { - super(clusteringType, fullType); - this.columnNameType = columnNameType; - this.internedIds = internedIds; - this.staticPrefix = makeStaticPrefix(clusteringType.size()); - } - - private static Composite makeStaticPrefix(int size) - { - ByteBuffer[] elements = new ByteBuffer[size]; - for (int i = 0; i < size; i++) - elements[i] = ByteBufferUtil.EMPTY_BYTE_BUFFER; - - return new CompoundComposite(elements, size, true) - { - @Override - public boolean isStatic() - { - return true; - } - - @Override - public long unsharedHeapSize() - { - // We'll share this for a given type. - return 0; - } - - @Override - public Composite copy(CFMetaData cfm, AbstractAllocator allocator) - { - return this; - } - }; - } - - protected static CompoundCType makeCType(CompoundCType clusteringType, AbstractType columnNameType, ColumnToCollectionType collectionType) - { - List> allSubtypes = new ArrayList>(clusteringType.size() + (collectionType == null ? 1 : 2)); - for (int i = 0; i < clusteringType.size(); i++) - allSubtypes.add(clusteringType.subtype(i)); - allSubtypes.add(columnNameType); - if (collectionType != null) - allSubtypes.add(collectionType); - return new CompoundCType(allSubtypes); - } - - public CellNameType setSubtype(int position, AbstractType newType) - { - if (position < clusteringSize) - return new CompoundSparseCellNameType(clusteringType.setSubtype(position, newType), columnNameType, fullType.setSubtype(position, newType), internedIds); - - if (position == clusteringSize) - throw new IllegalArgumentException(); - - throw new IndexOutOfBoundsException(); - } - - @Override - public CellNameType addOrUpdateCollection(ColumnIdentifier columnName, CollectionType newCollection) - { - return new WithCollection(clusteringType, ColumnToCollectionType.getInstance(Collections.singletonMap(columnName.bytes, newCollection)), internedIds); - } - - public boolean isDense() - { - return false; - } - - public boolean supportCollections() - { - return true; - } - - public Composite staticPrefix() - { - return staticPrefix; - } - - public CellName create(Composite prefix, ColumnDefinition column) - { - return create(prefix, column.name, column.isStatic()); - } - - private CellName create(Composite prefix, ColumnIdentifier columnName, boolean isStatic) - { - if (isStatic) - prefix = staticPrefix(); - - assert prefix.size() == clusteringSize; - - if (prefix.isEmpty()) - return new CompoundSparseCellName(columnName, isStatic); - - assert prefix instanceof CompoundComposite; - CompoundComposite lc = (CompoundComposite)prefix; - return new CompoundSparseCellName(lc.elements, clusteringSize, columnName, isStatic); - } - - public CellName rowMarker(Composite prefix) - { - assert !prefix.isStatic(); // static columns don't really create rows, they shouldn't have a row marker - if (prefix.isEmpty()) - return rowMarkerNoPrefix; - - return create(prefix, rowMarkerId, false); - } - - protected ColumnIdentifier idFor(ByteBuffer bb) - { - ColumnIdentifier id = internedIds.get(bb); - return id == null ? new ColumnIdentifier(bb, columnNameType) : id; - } - - protected Composite makeWith(ByteBuffer[] components, int size, Composite.EOC eoc, boolean isStatic) - { - if (size < clusteringSize + 1 || eoc != Composite.EOC.NONE) - return new CompoundComposite(components, size, isStatic).withEOC(eoc); - - return new CompoundSparseCellName(components, clusteringSize, idFor(components[clusteringSize]), isStatic); - } - - protected Composite copyAndMakeWith(ByteBuffer[] components, int size, Composite.EOC eoc, boolean isStatic) - { - if (size < clusteringSize + 1 || eoc != Composite.EOC.NONE) - return new CompoundComposite(Arrays.copyOfRange(components, 0, size), size, isStatic).withEOC(eoc); - - ByteBuffer[] clusteringColumns = Arrays.copyOfRange(components, 0, clusteringSize); - return new CompoundSparseCellName(clusteringColumns, idFor(components[clusteringSize]), isStatic); - } - - public void addCQL3Column(ColumnIdentifier id) - { - internedIds.put(id.bytes, id); - } - - public void removeCQL3Column(ColumnIdentifier id) - { - internedIds.remove(id.bytes); - } - - public CQL3Row.Builder CQL3RowBuilder(CFMetaData metadata, long now) - { - return makeSparseCQL3RowBuilder(metadata, this, now); - } - - public static class WithCollection extends CompoundSparseCellNameType - { - private final ColumnToCollectionType collectionType; - - public WithCollection(List> types, ColumnToCollectionType collectionType) - { - this(new CompoundCType(types), collectionType); - } - - WithCollection(CompoundCType clusteringType, ColumnToCollectionType collectionType) - { - this(clusteringType, collectionType, new HashMap()); - } - - private WithCollection(CompoundCType clusteringType, ColumnToCollectionType collectionType, Map internedIds) - { - this(clusteringType, makeCType(clusteringType, UTF8Type.instance, collectionType), collectionType, internedIds); - } - - private WithCollection(CompoundCType clusteringType, CompoundCType fullCType, ColumnToCollectionType collectionType, Map internedIds) - { - super(clusteringType, UTF8Type.instance, fullCType, internedIds); - this.collectionType = collectionType; - } - - @Override - public CellNameType setSubtype(int position, AbstractType newType) - { - if (position < clusteringSize) - return new WithCollection(clusteringType.setSubtype(position, newType), collectionType, internedIds); - - throw position >= fullType.size() ? new IndexOutOfBoundsException() : new IllegalArgumentException(); - } - - @Override - public CellNameType addOrUpdateCollection(ColumnIdentifier columnName, CollectionType newCollection) - { - Map newMap = new HashMap<>(collectionType.defined); - newMap.put(columnName.bytes, newCollection); - return new WithCollection(clusteringType, ColumnToCollectionType.getInstance(newMap), internedIds); - } - - @Override - public CellName create(Composite prefix, ColumnDefinition column, ByteBuffer collectionElement) - { - if (column.isStatic()) - prefix = staticPrefix(); - - assert prefix.size() == clusteringSize; - - if (prefix.isEmpty()) - return new CompoundSparseCellName.WithCollection(column.name, collectionElement, column.isStatic()); - - assert prefix instanceof CompoundComposite; - CompoundComposite lc = (CompoundComposite)prefix; - return new CompoundSparseCellName.WithCollection(lc.elements, clusteringSize, column.name, collectionElement, column.isStatic()); - } - - @Override - public int compare(Composite c1, Composite c2) - { - if (c1.isStatic() != c2.isStatic()) - { - // Static sorts before non-static no matter what, except for empty which - // always sort first - if (c1.isEmpty()) - return c2.isEmpty() ? 0 : -1; - if (c2.isEmpty()) - return 1; - return c1.isStatic() ? -1 : 1; - } - - int s1 = c1.size(); - int s2 = c2.size(); - int minSize = Math.min(s1, s2); - - ByteBuffer previous = null; - for (int i = 0; i < minSize; i++) - { - AbstractType comparator = subtype(i); - ByteBuffer value1 = c1.get(i); - ByteBuffer value2 = c2.get(i); - - int cmp = comparator.compareCollectionMembers(value1, value2, previous); - if (cmp != 0) - return cmp; - - previous = value1; - } - - if (s1 == s2) - return c1.eoc().compareTo(c2.eoc()); - return s1 < s2 ? c1.eoc().prefixComparisonResult : -c2.eoc().prefixComparisonResult; - } - - @Override - public boolean hasCollections() - { - return true; - } - - @Override - public ColumnToCollectionType collectionType() - { - return collectionType; - } - - @Override - protected Composite makeWith(ByteBuffer[] components, int size, Composite.EOC eoc, boolean isStatic) - { - if (size < fullSize) - return super.makeWith(components, size, eoc, isStatic); - - return new CompoundSparseCellName.WithCollection(components, clusteringSize, idFor(components[clusteringSize]), components[fullSize - 1], isStatic); - } - - protected Composite copyAndMakeWith(ByteBuffer[] components, int size, Composite.EOC eoc, boolean isStatic) - { - if (size < fullSize) - return super.copyAndMakeWith(components, size, eoc, isStatic); - - ByteBuffer[] clusteringColumns = Arrays.copyOfRange(components, 0, clusteringSize); - return new CompoundSparseCellName.WithCollection(clusteringColumns, idFor(components[clusteringSize]), components[clusteringSize + 1], isStatic); - } - } -} - diff --git a/src/java/org/apache/cassandra/db/composites/SimpleCType.java b/src/java/org/apache/cassandra/db/composites/SimpleCType.java deleted file mode 100644 index 7ee45acdec..0000000000 --- a/src/java/org/apache/cassandra/db/composites/SimpleCType.java +++ /dev/null @@ -1,156 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.apache.cassandra.db.composites; - -import java.nio.ByteBuffer; -import java.util.List; - -import org.apache.cassandra.db.marshal.AbstractType; - -/** - * A not truly-composite CType. - */ -public class SimpleCType extends AbstractCType -{ - protected final AbstractType type; - - public SimpleCType(AbstractType type) - { - super(type.isByteOrderComparable()); - this.type = type; - } - - public boolean isCompound() - { - return false; - } - - public int size() - { - return 1; - } - - public int compare(Composite c1, Composite c2) - { - if (isByteOrderComparable) - return AbstractSimpleCellNameType.compareUnsigned(c1, c2); - - assert !(c1.isEmpty() | c2.isEmpty()); - // This method assumes that simple composites never have an EOC != NONE. This assumption - // stands in particular on the fact that a Composites.EMPTY never has a non-NONE EOC. If - // this ever change, we'll need to update this. - return type.compare(c1.get(0), c2.get(0)); - } - - public AbstractType subtype(int i) - { - if (i != 0) - throw new IndexOutOfBoundsException(); - return type; - } - - public Composite fromByteBuffer(ByteBuffer bytes) - { - return !bytes.hasRemaining() ? Composites.EMPTY : new SimpleComposite(bytes); - } - - public CBuilder builder() - { - return new SimpleCBuilder(this); - } - - public CType setSubtype(int position, AbstractType newType) - { - if (position != 0) - throw new IndexOutOfBoundsException(); - return new SimpleCType(newType); - } - - // Use sparingly, it defeats the purpose - public AbstractType asAbstractType() - { - return type; - } - - public static class SimpleCBuilder implements CBuilder - { - private final CType type; - private ByteBuffer value; - - public SimpleCBuilder(CType type) - { - this.type = type; - } - - public int remainingCount() - { - return value == null ? 1 : 0; - } - - public CBuilder add(ByteBuffer value) - { - if (this.value != null) - throw new IllegalStateException(); - this.value = value; - return this; - } - - public CBuilder add(Object value) - { - return add(((AbstractType)type.subtype(0)).decompose(value)); - } - - public Composite build() - { - if (value == null || !value.hasRemaining()) - return Composites.EMPTY; - - // If we're building a dense cell name, then we can directly allocate the - // CellName object as it's complete. - if (type instanceof CellNameType && ((CellNameType)type).isDense()) - return new SimpleDenseCellName(value); - - return new SimpleComposite(value); - } - - public Composite buildWith(ByteBuffer value) - { - if (this.value != null) - throw new IllegalStateException(); - - if (value == null || !value.hasRemaining()) - return Composites.EMPTY; - - // If we're building a dense cell name, then we can directly allocate the - // CellName object as it's complete. - if (type instanceof CellNameType && ((CellNameType)type).isDense()) - return new SimpleDenseCellName(value); - - return new SimpleComposite(value); - } - - public Composite buildWith(List values) - { - if (values.size() > 1) - throw new IllegalStateException(); - if (values.isEmpty()) - return Composites.EMPTY; - return buildWith(values.get(0)); - } - } -} diff --git a/src/java/org/apache/cassandra/db/composites/SimpleComposite.java b/src/java/org/apache/cassandra/db/composites/SimpleComposite.java deleted file mode 100644 index 3c80d9f8cf..0000000000 --- a/src/java/org/apache/cassandra/db/composites/SimpleComposite.java +++ /dev/null @@ -1,79 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.apache.cassandra.db.composites; - -import java.nio.ByteBuffer; - -import org.apache.cassandra.config.CFMetaData; -import org.apache.cassandra.utils.memory.AbstractAllocator; -import org.apache.cassandra.utils.ObjectSizes; - -/** - * A "simple" (not-truly-composite) Composite. - */ -public class SimpleComposite extends AbstractComposite -{ - private static final long EMPTY_SIZE = ObjectSizes.measure(new SimpleComposite(ByteBuffer.allocate(1))); - - protected final ByteBuffer element; - - SimpleComposite(ByteBuffer element) - { - // We have to be careful with empty ByteBuffers as we shouldn't store them. - // To avoid errors (and so isEmpty() works as we intend), we don't allow simpleComposite with - // an empty element (but it's ok for CompoundComposite, it's a row marker in that case). - assert element.hasRemaining(); - this.element = element; - } - - public int size() - { - return 1; - } - - public ByteBuffer get(int i) - { - if (i != 0) - throw new IndexOutOfBoundsException(); - - return element; - } - - @Override - public Composite withEOC(EOC newEoc) - { - // EOC makes no sense for not truly composites. - return this; - } - - @Override - public ByteBuffer toByteBuffer() - { - return element; - } - - public long unsharedHeapSize() - { - return EMPTY_SIZE + ObjectSizes.sizeOnHeapOf(element); - } - - public Composite copy(CFMetaData cfm, AbstractAllocator allocator) - { - return new SimpleComposite(allocator.clone(element)); - } -} diff --git a/src/java/org/apache/cassandra/db/composites/SimpleDenseCellName.java b/src/java/org/apache/cassandra/db/composites/SimpleDenseCellName.java deleted file mode 100644 index 2ca7d23353..0000000000 --- a/src/java/org/apache/cassandra/db/composites/SimpleDenseCellName.java +++ /dev/null @@ -1,83 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.apache.cassandra.db.composites; - -import java.nio.ByteBuffer; - -import org.apache.cassandra.config.CFMetaData; -import org.apache.cassandra.cql3.ColumnIdentifier; -import org.apache.cassandra.utils.memory.AbstractAllocator; -import org.apache.cassandra.utils.ObjectSizes; - -public class SimpleDenseCellName extends SimpleComposite implements CellName -{ - private static final long EMPTY_SIZE = ObjectSizes.measure(new SimpleDenseCellName(ByteBuffer.allocate(1))); - - // Not meant to be used directly, you should use the CellNameType method instead - SimpleDenseCellName(ByteBuffer element) - { - super(element); - } - - public int clusteringSize() - { - return 1; - } - - public ColumnIdentifier cql3ColumnName(CFMetaData metadata) - { - return null; - } - - public ByteBuffer collectionElement() - { - return null; - } - - public boolean isCollectionCell() - { - return false; - } - - public boolean isSameCQL3RowAs(CellNameType type, CellName other) - { - // Dense cell imply one cell by CQL row so no other cell will be the same row. - return type.compare(this, other) == 0; - } - - @Override - public long unsharedHeapSize() - { - return EMPTY_SIZE + ObjectSizes.sizeOnHeapOf(element); - } - - @Override - public long unsharedHeapSizeExcludingData() - { - return EMPTY_SIZE + ObjectSizes.sizeOnHeapExcludingData(element); - } - - // If cellnames were sharing some prefix components, this will break it, so - // we might want to try to do better. - @Override - public CellName copy(CFMetaData cfm, AbstractAllocator allocator) - { - return new SimpleDenseCellName(allocator.clone(element)); - } - -} diff --git a/src/java/org/apache/cassandra/db/composites/SimpleDenseCellNameType.java b/src/java/org/apache/cassandra/db/composites/SimpleDenseCellNameType.java deleted file mode 100644 index 3db4bc439d..0000000000 --- a/src/java/org/apache/cassandra/db/composites/SimpleDenseCellNameType.java +++ /dev/null @@ -1,80 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.apache.cassandra.db.composites; - -import java.nio.ByteBuffer; - -import org.apache.cassandra.config.CFMetaData; -import org.apache.cassandra.config.ColumnDefinition; -import org.apache.cassandra.cql3.CQL3Row; -import org.apache.cassandra.cql3.ColumnIdentifier; -import org.apache.cassandra.db.marshal.AbstractType; - -public class SimpleDenseCellNameType extends AbstractSimpleCellNameType -{ - public SimpleDenseCellNameType(AbstractType type) - { - super(type); - } - - public int clusteringPrefixSize() - { - return 1; - } - - public CBuilder prefixBuilder() - { - // Simple dense is "all" prefix - return builder(); - } - - public CellNameType setSubtype(int position, AbstractType newType) - { - if (position != 0) - throw new IllegalArgumentException(); - return new SimpleDenseCellNameType(newType); - } - - public boolean isDense() - { - return true; - } - - public CellName create(Composite prefix, ColumnDefinition column) - { - assert prefix.size() == 1; - // We ignore the column because it's just the COMPACT_VALUE name which is not store in the cell name - return new SimpleDenseCellName(prefix.get(0)); - } - - @Override - public Composite fromByteBuffer(ByteBuffer bb) - { - return !bb.hasRemaining() - ? Composites.EMPTY - : new SimpleDenseCellName(bb); - } - - public void addCQL3Column(ColumnIdentifier id) {} - public void removeCQL3Column(ColumnIdentifier id) {} - - public CQL3Row.Builder CQL3RowBuilder(CFMetaData metadata, long now) - { - return makeDenseCQL3RowBuilder(now); - } -} diff --git a/src/java/org/apache/cassandra/db/composites/SimpleSparseCellName.java b/src/java/org/apache/cassandra/db/composites/SimpleSparseCellName.java deleted file mode 100644 index c6351f1dbd..0000000000 --- a/src/java/org/apache/cassandra/db/composites/SimpleSparseCellName.java +++ /dev/null @@ -1,104 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.apache.cassandra.db.composites; - -import java.nio.ByteBuffer; - -import org.apache.cassandra.config.CFMetaData; -import org.apache.cassandra.cql3.ColumnIdentifier; -import org.apache.cassandra.utils.memory.AbstractAllocator; -import org.apache.cassandra.utils.ObjectSizes; - -public class SimpleSparseCellName extends AbstractComposite implements CellName -{ - private static final long EMPTY_SIZE = ObjectSizes.measure(new SimpleSparseCellName(null)); - - private final ColumnIdentifier columnName; - - // Not meant to be used directly, you should use the CellNameType method instead - SimpleSparseCellName(ColumnIdentifier columnName) - { - this.columnName = columnName; - } - - public int size() - { - return 1; - } - - public ByteBuffer get(int i) - { - if (i != 0) - throw new IndexOutOfBoundsException(); - - return columnName.bytes; - } - - @Override - public Composite withEOC(EOC newEoc) - { - // EOC makes no sense for not truly composites. - return this; - } - - @Override - public ByteBuffer toByteBuffer() - { - return columnName.bytes; - } - - public int clusteringSize() - { - return 0; - } - - public ColumnIdentifier cql3ColumnName(CFMetaData metadata) - { - return columnName; - } - - public ByteBuffer collectionElement() - { - return null; - } - - public boolean isCollectionCell() - { - return false; - } - - public boolean isSameCQL3RowAs(CellNameType type, CellName other) - { - return true; - } - - public long unsharedHeapSizeExcludingData() - { - return EMPTY_SIZE + columnName.unsharedHeapSizeExcludingData(); - } - - public long unsharedHeapSize() - { - return EMPTY_SIZE + columnName.unsharedHeapSize(); - } - - public CellName copy(CFMetaData cfm, AbstractAllocator allocator) - { - return new SimpleSparseCellName(columnName.clone(allocator)); - } -} diff --git a/src/java/org/apache/cassandra/db/composites/SimpleSparseCellNameType.java b/src/java/org/apache/cassandra/db/composites/SimpleSparseCellNameType.java deleted file mode 100644 index 5ce0deb912..0000000000 --- a/src/java/org/apache/cassandra/db/composites/SimpleSparseCellNameType.java +++ /dev/null @@ -1,100 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.apache.cassandra.db.composites; - -import java.nio.ByteBuffer; -import java.util.HashMap; -import java.util.Map; - -import org.apache.cassandra.config.CFMetaData; -import org.apache.cassandra.config.ColumnDefinition; -import org.apache.cassandra.cql3.CQL3Row; -import org.apache.cassandra.cql3.ColumnIdentifier; -import org.apache.cassandra.db.marshal.AbstractType; - -public class SimpleSparseCellNameType extends AbstractSimpleCellNameType -{ - // Simple sparse means static thrift CF or non-clustered CQL3. This means that cell names will mainly - // be those that have been declared and we can intern the whole CellName instances. - private final Map internedNames; - - public SimpleSparseCellNameType(AbstractType type) - { - this(type, new HashMap()); - } - - private SimpleSparseCellNameType(AbstractType type, Map internedNames) - { - super(type); - this.internedNames = internedNames; - } - - public int clusteringPrefixSize() - { - return 0; - } - - public CellNameType setSubtype(int position, AbstractType newType) - { - if (position != 0) - throw new IllegalArgumentException(); - return new SimpleSparseCellNameType(newType, internedNames); - } - - public CBuilder prefixBuilder() - { - return Composites.EMPTY_BUILDER; - } - - public boolean isDense() - { - return false; - } - - public CellName create(Composite prefix, ColumnDefinition column) - { - assert prefix.isEmpty(); - CellName cn = internedNames.get(column.name.bytes); - return cn == null ? new SimpleSparseCellName(column.name) : cn; - } - - @Override - public Composite fromByteBuffer(ByteBuffer bb) - { - if (!bb.hasRemaining()) - return Composites.EMPTY; - - CellName cn = internedNames.get(bb); - return cn == null ? new SimpleSparseCellName(new ColumnIdentifier(bb, type)) : cn; - } - - public void addCQL3Column(ColumnIdentifier id) - { - internedNames.put(id.bytes, new SimpleSparseInternedCellName(id)); - } - - public void removeCQL3Column(ColumnIdentifier id) - { - internedNames.remove(id.bytes); - } - - public CQL3Row.Builder CQL3RowBuilder(CFMetaData metadata, long now) - { - return makeSparseCQL3RowBuilder(metadata, this, now); - } -} diff --git a/src/java/org/apache/cassandra/db/context/CounterContext.java b/src/java/org/apache/cassandra/db/context/CounterContext.java index ffffbb14a5..2a6c5ff468 100644 --- a/src/java/org/apache/cassandra/db/context/CounterContext.java +++ b/src/java/org/apache/cassandra/db/context/CounterContext.java @@ -111,7 +111,12 @@ public class CounterContext /** * Creates a counter context with a single local shard. - * For use by tests of compatibility with pre-2.1 counters only. + * This is only used in a PartitionUpdate until the update has gone through + * CounterMutation.apply(), at which point all the local shard are replaced by + * global ones. In other words, local shards should never hit the disk or + * memtables. And we use this so that if an update statement has multiple increment + * of the same counter we properly add them rather than keeping only one of them. + * (this is also used for tests of compatibility with pre-2.1 counters) */ public ByteBuffer createLocal(long count) { diff --git a/src/java/org/apache/cassandra/db/filter/AbstractClusteringIndexFilter.java b/src/java/org/apache/cassandra/db/filter/AbstractClusteringIndexFilter.java new file mode 100644 index 0000000000..46d10df8e7 --- /dev/null +++ b/src/java/org/apache/cassandra/db/filter/AbstractClusteringIndexFilter.java @@ -0,0 +1,110 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.cassandra.db.filter; + +import java.io.DataInput; +import java.io.IOException; + +import org.apache.cassandra.config.CFMetaData; +import org.apache.cassandra.config.ColumnDefinition; +import org.apache.cassandra.db.*; +import org.apache.cassandra.db.marshal.ReversedType; +import org.apache.cassandra.io.util.DataOutputPlus; + +public abstract class AbstractClusteringIndexFilter implements ClusteringIndexFilter +{ + protected enum Kind + { + SLICE (ClusteringIndexSliceFilter.deserializer), + NAMES (ClusteringIndexNamesFilter.deserializer); + + private final InternalDeserializer deserializer; + + private Kind(InternalDeserializer deserializer) + { + this.deserializer = deserializer; + } + } + + static final Serializer serializer = new FilterSerializer(); + + abstract Kind kind(); + + protected final boolean reversed; + + protected AbstractClusteringIndexFilter(boolean reversed) + { + this.reversed = reversed; + } + + public boolean isReversed() + { + return reversed; + } + + protected abstract void serializeInternal(DataOutputPlus out, int version) throws IOException; + protected abstract long serializedSizeInternal(int version, TypeSizes sizes); + + protected void appendOrderByToCQLString(CFMetaData metadata, StringBuilder sb) + { + if (reversed) + { + sb.append(" ORDER BY ("); + int i = 0; + for (ColumnDefinition column : metadata.clusteringColumns()) + sb.append(i++ == 0 ? "" : ", ").append(column.name).append(column.type instanceof ReversedType ? " ASC" : " DESC"); + sb.append(")"); + } + } + + private static class FilterSerializer implements Serializer + { + public void serialize(ClusteringIndexFilter pfilter, DataOutputPlus out, int version) throws IOException + { + AbstractClusteringIndexFilter filter = (AbstractClusteringIndexFilter)pfilter; + + out.writeByte(filter.kind().ordinal()); + out.writeBoolean(filter.isReversed()); + + filter.serializeInternal(out, version); + } + + public ClusteringIndexFilter deserialize(DataInput in, int version, CFMetaData metadata) throws IOException + { + Kind kind = Kind.values()[in.readUnsignedByte()]; + boolean reversed = in.readBoolean(); + + return kind.deserializer.deserialize(in, version, metadata, reversed); + } + + public long serializedSize(ClusteringIndexFilter pfilter, int version) + { + AbstractClusteringIndexFilter filter = (AbstractClusteringIndexFilter)pfilter; + + TypeSizes sizes = TypeSizes.NATIVE; + return 1 + + sizes.sizeof(filter.isReversed()) + + filter.serializedSizeInternal(version, sizes); + } + } + + protected static abstract class InternalDeserializer + { + public abstract ClusteringIndexFilter deserialize(DataInput in, int version, CFMetaData metadata, boolean reversed) throws IOException; + } +} diff --git a/src/java/org/apache/cassandra/db/filter/ClusteringIndexFilter.java b/src/java/org/apache/cassandra/db/filter/ClusteringIndexFilter.java new file mode 100644 index 0000000000..54feb85810 --- /dev/null +++ b/src/java/org/apache/cassandra/db/filter/ClusteringIndexFilter.java @@ -0,0 +1,152 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.cassandra.db.filter; + +import java.io.DataInput; +import java.io.IOException; + +import org.apache.cassandra.config.CFMetaData; +import org.apache.cassandra.db.*; +import org.apache.cassandra.db.rows.*; +import org.apache.cassandra.db.partitions.CachedPartition; +import org.apache.cassandra.db.partitions.Partition; +import org.apache.cassandra.io.sstable.format.SSTableReader; +import org.apache.cassandra.io.util.DataOutputPlus; + +/** + * A filter that selects a subset of the rows of a given partition by using the "clustering index". + *

+ * In CQL terms, this correspond to the clustering columns selection and correspond to what + * the storage engine can do without filtering (and without 2ndary indexes). This does not include + * the restrictions on non-PK columns which can be found in {@link RowFilter}. + */ +public interface ClusteringIndexFilter +{ + public static Serializer serializer = AbstractClusteringIndexFilter.serializer; + + /** + * Whether the filter query rows in reversed clustering order or not. + * + * @return whether the filter query rows in reversed clustering order or not. + */ + public boolean isReversed(); + + /** + * Returns a filter for continuing the paging of this filter given the last returned clustering prefix. + * + * @param comparator the comparator for the table this is a filter for. + * @param lastReturned the last clustering that was returned for the query we are paging for. The + * resulting filter will be such that results coming after {@code lastReturned} are returned + * (where coming after means "greater than" if the filter is not reversed, "lesser than" otherwise; + * futher, whether the comparison is strict or not depends on {@code inclusive}). + * @param inclusive whether or not we want to include the {@code lastReturned} in the newly returned + * page of results. + * + * @return a new filter that selects results coming after {@code lastReturned}. + */ + public ClusteringIndexFilter forPaging(ClusteringComparator comparator, Clustering lastReturned, boolean inclusive); + + /** + * Returns whether we can guarantee that a given cached partition contains all the data selected by this filter. + * + * @param partition the cached partition. This method assumed that the rows of this partition contains all the table columns. + * + * @return whether we can guarantee that all data selected by this filter are in {@code partition}. + */ + public boolean isFullyCoveredBy(CachedPartition partition); + + /** + * Whether this filter selects the head of a partition (i.e. it isn't reversed and selects all rows up to a certain point). + * + * @return whether this filter selects the head of a partition. + */ + public boolean isHeadFilter(); + + /** + * Whether this filter selects all the row of a partition (it's an "identity" filter). + * + * @return whether this filter selects all the row of a partition (it's an "identity" filter). + */ + public boolean selectsAllPartition(); + + /** + * Whether a given row is selected by this filter. + * + * @param clustering the clustering of the row to test the selection of. + * + * @return whether the row with clustering {@code clustering} is selected by this filter. + */ + public boolean selects(Clustering clustering); + + /** + * Returns an iterator that only returns the rows of the provided iterator that this filter selects. + *

+ * This method is the "dumb" counterpart to {@link #filter(SliceableUnfilteredRowIterator)} in that it has no way to quickly get + * to what is actually selected, so it simply iterate over it all and filters out what shouldn't be returned. This should + * be avoided in general, we should make sure to have {@code SliceableUnfilteredRowIterator} when we have filtering to do, but this + * currently only used in {@link SinglePartitionReadCommand#getThroughCache} when we know this won't be a performance problem. + * Another difference with {@link #filter(SliceableUnfilteredRowIterator)} is that this method also filter the queried + * columns in the returned result, while the former assumes that the provided iterator has already done it. + * + * @param columnFilter the columns to include in the rows of the result iterator. + * @param iterator the iterator for which we should filter rows. + * + * @return an iterator that only returns the rows (or rather Unfilted) from {@code iterator} that are selected by this filter. + */ + public UnfilteredRowIterator filterNotIndexed(ColumnFilter columnFilter, UnfilteredRowIterator iterator); + + /** + * Returns an iterator that only returns the rows of the provided sliceable iterator that this filter selects. + * + * @param iterator the sliceable iterator for which we should filter rows. + * + * @return an iterator that only returns the rows (or rather unfiltered) from {@code iterator} that are selected by this filter. + */ + public UnfilteredRowIterator filter(SliceableUnfilteredRowIterator iterator); + + /** + * Given a partition, returns a row iterator for the rows of this partition that are selected by this filter. + * + * @param columnFilter the columns to include in the rows of the result iterator. + * @param partition the partition containing the rows to filter. + * + * @return a unfiltered row iterator returning those rows (or rather Unfiltered) from {@code partition} that are selected by this filter. + */ + // TODO: we could get rid of that if Partition was exposing a SliceableUnfilteredRowIterator (instead of the two searchIterator() and + // unfilteredIterator() methods). However, for AtomicBtreePartition this would require changes to Btree so we'll leave that for later. + public UnfilteredRowIterator getUnfilteredRowIterator(ColumnFilter columnFilter, Partition partition); + + /** + * Whether the provided sstable may contain data that is selected by this filter (based on the sstable metadata). + * + * @param sstable the sstable for which we want to test the need for inclusion. + * + * @return whether {@code sstable} should be included to answer this filter. + */ + public boolean shouldInclude(SSTableReader sstable); + + public String toString(CFMetaData metadata); + public String toCQLString(CFMetaData metadata); + + public interface Serializer + { + public void serialize(ClusteringIndexFilter filter, DataOutputPlus out, int version) throws IOException; + public ClusteringIndexFilter deserialize(DataInput in, int version, CFMetaData metadata) throws IOException; + public long serializedSize(ClusteringIndexFilter filter, int version); + } +} diff --git a/src/java/org/apache/cassandra/db/filter/ClusteringIndexNamesFilter.java b/src/java/org/apache/cassandra/db/filter/ClusteringIndexNamesFilter.java new file mode 100644 index 0000000000..1839d3e20d --- /dev/null +++ b/src/java/org/apache/cassandra/db/filter/ClusteringIndexNamesFilter.java @@ -0,0 +1,271 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.cassandra.db.filter; + +import java.io.DataInput; +import java.io.IOException; +import java.util.*; + +import org.apache.cassandra.config.CFMetaData; +import org.apache.cassandra.config.ColumnDefinition; +import org.apache.cassandra.db.*; +import org.apache.cassandra.db.rows.*; +import org.apache.cassandra.db.partitions.*; +import org.apache.cassandra.io.sstable.format.SSTableReader; +import org.apache.cassandra.io.util.DataOutputPlus; +import org.apache.cassandra.utils.SearchIterator; + +/** + * A filter selecting rows given their clustering value. + */ +public class ClusteringIndexNamesFilter extends AbstractClusteringIndexFilter +{ + static final InternalDeserializer deserializer = new NamesDeserializer(); + + // This could be empty if selectedColumns only has static columns (in which case the filter still + // selects the static row) + private final NavigableSet clusterings; + + // clusterings is always in clustering order (because we need it that way in some methods), but we also + // sometimes need those clustering in "query" order (i.e. in reverse clustering order if the query is + // reversed), so we keep that too for simplicity. + private final NavigableSet clusteringsInQueryOrder; + + public ClusteringIndexNamesFilter(NavigableSet clusterings, boolean reversed) + { + super(reversed); + assert !clusterings.contains(Clustering.STATIC_CLUSTERING); + this.clusterings = clusterings; + this.clusteringsInQueryOrder = reversed ? clusterings.descendingSet() : clusterings; + } + + /** + * The set of requested rows. + * + * Please note that this can be empty if only the static row is requested. + * + * @return the set of requested clustering in clustering order (note that + * this is always in clustering order even if the query is reversed). + */ + public NavigableSet requestedRows() + { + return clusterings; + } + + public boolean selectsAllPartition() + { + return false; + } + + public boolean selects(Clustering clustering) + { + return clusterings.contains(clustering); + } + + public ClusteringIndexNamesFilter forPaging(ClusteringComparator comparator, Clustering lastReturned, boolean inclusive) + { + // TODO: Consider removal of the initial check. + int cmp = comparator.compare(lastReturned, clusteringsInQueryOrder.first()); + if (cmp < 0 || (inclusive && cmp == 0)) + return this; + + NavigableSet newClusterings = reversed ? + clusterings.headSet(lastReturned, inclusive) : + clusterings.tailSet(lastReturned, inclusive); + + return new ClusteringIndexNamesFilter(newClusterings, reversed); + } + + public boolean isFullyCoveredBy(CachedPartition partition) + { + // 'partition' contains all columns, so it covers our filter if our last clusterings + // is smaller than the last in the cache + return clusterings.comparator().compare(clusterings.last(), partition.lastRow().clustering()) <= 0; + } + + public boolean isHeadFilter() + { + return false; + } + + // Given another iterator, only return the rows that match this filter + public UnfilteredRowIterator filterNotIndexed(ColumnFilter columnFilter, UnfilteredRowIterator iterator) + { + // Note that we don't filter markers because that's a bit trickier (we don't know in advance until when + // the range extend) and it's harmless to left them. + return new FilteringRowIterator(iterator) + { + @Override + public FilteringRow makeRowFilter() + { + return FilteringRow.columnsFilteringRow(columnFilter); + } + + @Override + protected boolean includeRow(Row row) + { + return clusterings.contains(row.clustering()); + } + }; + } + + public UnfilteredRowIterator filter(final SliceableUnfilteredRowIterator iter) + { + // Please note that this method assumes that rows from 'iter' already have their columns filtered, i.e. that + // they only include columns that we select. + return new WrappingUnfilteredRowIterator(iter) + { + private final Iterator clusteringIter = clusteringsInQueryOrder.iterator(); + private Iterator currentClustering; + private Unfiltered next; + + @Override + public boolean hasNext() + { + if (next != null) + return true; + + if (currentClustering != null && currentClustering.hasNext()) + { + next = currentClustering.next(); + return true; + } + + while (clusteringIter.hasNext()) + { + Clustering nextClustering = clusteringIter.next(); + currentClustering = iter.slice(Slice.make(nextClustering)); + if (currentClustering.hasNext()) + { + next = currentClustering.next(); + return true; + } + } + return false; + } + + @Override + public Unfiltered next() + { + if (next == null && !hasNext()) + throw new NoSuchElementException(); + + Unfiltered toReturn = next; + next = null; + return toReturn; + } + }; + } + + public UnfilteredRowIterator getUnfilteredRowIterator(final ColumnFilter columnFilter, final Partition partition) + { + final SearchIterator searcher = partition.searchIterator(columnFilter, reversed); + return new AbstractUnfilteredRowIterator(partition.metadata(), + partition.partitionKey(), + partition.partitionLevelDeletion(), + columnFilter.fetchedColumns(), + searcher.next(Clustering.STATIC_CLUSTERING), + reversed, + partition.stats()) + { + private final Iterator clusteringIter = clusteringsInQueryOrder.iterator(); + + protected Unfiltered computeNext() + { + while (clusteringIter.hasNext() && searcher.hasNext()) + { + Row row = searcher.next(clusteringIter.next()); + if (row != null) + return row; + } + return endOfData(); + } + }; + } + + public boolean shouldInclude(SSTableReader sstable) + { + // TODO: we could actually exclude some sstables + return true; + } + + public String toString(CFMetaData metadata) + { + StringBuilder sb = new StringBuilder(); + sb.append("names("); + int i = 0; + for (Clustering clustering : clusterings) + sb.append(i++ == 0 ? "" : ", ").append(clustering.toString(metadata)); + if (reversed) + sb.append(", reversed"); + return sb.append(")").toString(); + } + + public String toCQLString(CFMetaData metadata) + { + if (clusterings.isEmpty()) + return ""; + + StringBuilder sb = new StringBuilder(); + sb.append("(").append(ColumnDefinition.toCQLString(metadata.clusteringColumns())).append(")"); + sb.append(clusterings.size() == 1 ? " = " : " IN ("); + int i = 0; + for (Clustering clustering : clusterings) + sb.append(i++ == 0 ? "" : ", ").append("(").append(clustering.toCQLString(metadata)).append(")"); + sb.append(clusterings.size() == 1 ? "" : ")"); + + appendOrderByToCQLString(metadata, sb); + return sb.toString(); + } + + Kind kind() + { + return Kind.NAMES; + } + + protected void serializeInternal(DataOutputPlus out, int version) throws IOException + { + ClusteringComparator comparator = (ClusteringComparator)clusterings.comparator(); + out.writeInt(clusterings.size()); + for (Clustering clustering : clusterings) + Clustering.serializer.serialize(clustering, out, version, comparator.subtypes()); + } + + protected long serializedSizeInternal(int version, TypeSizes sizes) + { + long size = 0; + ClusteringComparator comparator = (ClusteringComparator)clusterings.comparator(); + for (Clustering clustering : clusterings) + size += Clustering.serializer.serializedSize(clustering, version, comparator.subtypes(), sizes); + return size; + } + + private static class NamesDeserializer extends InternalDeserializer + { + public ClusteringIndexFilter deserialize(DataInput in, int version, CFMetaData metadata, boolean reversed) throws IOException + { + ClusteringComparator comparator = metadata.comparator; + NavigableSet clusterings = new TreeSet<>(comparator); + int size = in.readInt(); + for (int i = 0; i < size; i++) + clusterings.add(Clustering.serializer.deserialize(in, version, comparator.subtypes()).takeAlias()); + + return new ClusteringIndexNamesFilter(clusterings, reversed); + } + } +} diff --git a/src/java/org/apache/cassandra/db/filter/ClusteringIndexSliceFilter.java b/src/java/org/apache/cassandra/db/filter/ClusteringIndexSliceFilter.java new file mode 100644 index 0000000000..9e58542a1d --- /dev/null +++ b/src/java/org/apache/cassandra/db/filter/ClusteringIndexSliceFilter.java @@ -0,0 +1,179 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.cassandra.db.filter; + +import java.io.DataInput; +import java.io.IOException; +import java.util.List; +import java.nio.ByteBuffer; + +import org.apache.cassandra.config.CFMetaData; +import org.apache.cassandra.db.*; +import org.apache.cassandra.db.rows.*; +import org.apache.cassandra.db.partitions.CachedPartition; +import org.apache.cassandra.db.partitions.Partition; +import org.apache.cassandra.io.sstable.format.SSTableReader; +import org.apache.cassandra.io.util.DataOutputPlus; + +/** + * A filter over a single partition. + */ +public class ClusteringIndexSliceFilter extends AbstractClusteringIndexFilter +{ + static final InternalDeserializer deserializer = new SliceDeserializer(); + + private final Slices slices; + + public ClusteringIndexSliceFilter(Slices slices, boolean reversed) + { + super(reversed); + this.slices = slices; + } + + public Slices requestedSlices() + { + return slices; + } + + public boolean selectsAllPartition() + { + return slices.size() == 1 && !slices.hasLowerBound() && !slices.hasUpperBound(); + } + + public boolean selects(Clustering clustering) + { + return slices.selects(clustering); + } + + public ClusteringIndexSliceFilter forPaging(ClusteringComparator comparator, Clustering lastReturned, boolean inclusive) + { + Slices newSlices = slices.forPaging(comparator, lastReturned, inclusive, reversed); + return slices == newSlices + ? this + : new ClusteringIndexSliceFilter(newSlices, reversed); + } + + public boolean isFullyCoveredBy(CachedPartition partition) + { + // Partition is guaranteed to cover the whole filter if it includes the filter start and finish bounds. + + // (note that since partition is the head of a partition, to have no lower bound is ok) + if (!slices.hasUpperBound() || partition.isEmpty()) + return false; + + return partition.metadata().comparator.compare(slices.get(slices.size() - 1).end(), partition.lastRow().clustering()) <= 0; + } + + public boolean isHeadFilter() + { + return !reversed && slices.size() == 1 && !slices.hasLowerBound(); + } + + // Given another iterator, only return the rows that match this filter + public UnfilteredRowIterator filterNotIndexed(final ColumnFilter columnFilter, UnfilteredRowIterator iterator) + { + final Slices.InOrderTester tester = slices.inOrderTester(reversed); + + // Note that we don't filter markers because that's a bit trickier (we don't know in advance until when + // the range extend) and it's harmless to leave them. + return new FilteringRowIterator(iterator) + { + @Override + public FilteringRow makeRowFilter() + { + return FilteringRow.columnsFilteringRow(columnFilter); + } + + @Override + protected boolean includeRow(Row row) + { + return tester.includes(row.clustering()); + } + + @Override + public boolean hasNext() + { + return !tester.isDone() && super.hasNext(); + } + }; + } + + public UnfilteredRowIterator filter(SliceableUnfilteredRowIterator iterator) + { + // Please note that this method assumes that rows from 'iter' already have their columns filtered, i.e. that + // they only include columns that we select. + return slices.makeSliceIterator(iterator); + } + + public UnfilteredRowIterator getUnfilteredRowIterator(ColumnFilter columnFilter, Partition partition) + { + return partition.unfilteredIterator(columnFilter, slices, reversed); + } + + public boolean shouldInclude(SSTableReader sstable) + { + List minClusteringValues = sstable.getSSTableMetadata().minClusteringValues; + List maxClusteringValues = sstable.getSSTableMetadata().maxClusteringValues; + + if (minClusteringValues.isEmpty() || maxClusteringValues.isEmpty()) + return true; + + return slices.intersects(minClusteringValues, maxClusteringValues); + } + + public String toString(CFMetaData metadata) + { + return String.format("slice(slices=%s, reversed=%b)", slices, reversed); + } + + public String toCQLString(CFMetaData metadata) + { + StringBuilder sb = new StringBuilder(); + + if (!selectsAllPartition()) + sb.append(slices.toCQLString(metadata)); + + appendOrderByToCQLString(metadata, sb); + + return sb.toString(); + } + + Kind kind() + { + return Kind.SLICE; + } + + protected void serializeInternal(DataOutputPlus out, int version) throws IOException + { + Slices.serializer.serialize(slices, out, version); + } + + protected long serializedSizeInternal(int version, TypeSizes sizes) + { + return Slices.serializer.serializedSize(slices, version, sizes); + } + + private static class SliceDeserializer extends InternalDeserializer + { + public ClusteringIndexFilter deserialize(DataInput in, int version, CFMetaData metadata, boolean reversed) throws IOException + { + Slices slices = Slices.serializer.deserialize(in, version, metadata); + return new ClusteringIndexSliceFilter(slices, reversed); + } + } +} diff --git a/src/java/org/apache/cassandra/db/filter/ColumnCounter.java b/src/java/org/apache/cassandra/db/filter/ColumnCounter.java deleted file mode 100644 index 0d5acd177d..0000000000 --- a/src/java/org/apache/cassandra/db/filter/ColumnCounter.java +++ /dev/null @@ -1,217 +0,0 @@ -/* - * - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - * - */ -package org.apache.cassandra.db.filter; - -import org.apache.cassandra.db.Cell; -import org.apache.cassandra.db.composites.CellName; -import org.apache.cassandra.db.composites.CellNameType; -import org.apache.cassandra.db.ColumnFamily; -import org.apache.cassandra.db.DeletionInfo; - -public class ColumnCounter -{ - protected int live; - protected int tombstones; - protected final long timestamp; - - public ColumnCounter(long timestamp) - { - this.timestamp = timestamp; - } - - /** - * @return true if the cell counted as a live cell or a valid tombstone; false if it got immediately discarded for - * being shadowed by a range- or a partition tombstone - */ - public boolean count(Cell cell, DeletionInfo.InOrderTester tester) - { - // The cell is shadowed by a higher-level deletion, and won't be retained. - // For the purposes of this counter, we don't care if it's a tombstone or not. - if (tester.isDeleted(cell)) - return false; - - if (cell.isLive(timestamp)) - live++; - else - tombstones++; - - return true; - } - - public int live() - { - return live; - } - - public int tombstones() - { - return tombstones; - } - - public ColumnCounter countAll(ColumnFamily container) - { - if (container == null) - return this; - - DeletionInfo.InOrderTester tester = container.inOrderDeletionTester(); - for (Cell c : container) - count(c, tester); - return this; - } - - public static class GroupByPrefix extends ColumnCounter - { - protected final CellNameType type; - protected final int toGroup; - protected CellName previous; - - /** - * A column counter that count only 1 for all the columns sharing a - * given prefix of the key. - * - * @param type the type of the column name. This can be null if {@code - * toGroup} is 0, otherwise it should be a composite. - * @param toGroup the number of composite components on which to group - * column. If 0, all columns are grouped, otherwise we group - * those for which the {@code toGroup} first component are equals. - */ - public GroupByPrefix(long timestamp, CellNameType type, int toGroup) - { - super(timestamp); - this.type = type; - this.toGroup = toGroup; - - assert toGroup == 0 || type != null; - } - - @Override - public boolean count(Cell cell, DeletionInfo.InOrderTester tester) - { - if (tester.isDeleted(cell)) - return false; - - if (!cell.isLive(timestamp)) - { - tombstones++; - return true; - } - - if (toGroup == 0) - { - live = 1; - return true; - } - - CellName current = cell.name(); - assert current.size() >= toGroup; - - if (previous != null) - { - boolean isSameGroup = previous.isStatic() == current.isStatic(); - if (isSameGroup) - { - for (int i = 0; i < toGroup; i++) - { - if (type.subtype(i).compare(previous.get(i), current.get(i)) != 0) - { - isSameGroup = false; - break; - } - } - } - - if (isSameGroup) - return true; - - // We want to count the static group as 1 (CQL) row only if it's the only - // group in the partition. So, since we have already counted it at this point, - // just don't count the 2nd group if there is one and the first one was static - if (previous.isStatic()) - { - previous = current; - return true; - } - } - - live++; - previous = current; - - return true; - } - } - - /** - * Similar to GroupByPrefix, but designed to handle counting cells in reverse order. - */ - public static class GroupByPrefixReversed extends GroupByPrefix - { - public GroupByPrefixReversed(long timestamp, CellNameType type, int toGroup) - { - super(timestamp, type, toGroup); - } - - @Override - public boolean count(Cell cell, DeletionInfo.InOrderTester tester) - { - if (tester.isDeleted(cell)) - return false; - - if (!cell.isLive(timestamp)) - { - tombstones++; - return true; - } - - if (toGroup == 0) - { - live = 1; - return true; - } - - CellName current = cell.name(); - assert current.size() >= toGroup; - - if (previous == null) - { - // This is the first group we've seen. If it happens to be static, we still want to increment the - // count because a) there are no-static rows (statics are always last in reversed order), and b) any - // static cells we see after this will not increment the count - previous = current; - live++; - } - else if (!current.isStatic()) // ignore statics if we've seen any other statics or any other groups - { - for (int i = 0; i < toGroup; i++) - { - if (type.subtype(i).compare(previous.get(i), current.get(i)) != 0) - { - // it's a new group - live++; - previous = current; - return true; - } - } - } - - return true; - } - } -} diff --git a/src/java/org/apache/cassandra/db/filter/ColumnFilter.java b/src/java/org/apache/cassandra/db/filter/ColumnFilter.java new file mode 100644 index 0000000000..99140ef2ba --- /dev/null +++ b/src/java/org/apache/cassandra/db/filter/ColumnFilter.java @@ -0,0 +1,437 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.cassandra.db.filter; + +import java.io.DataInput; +import java.io.IOException; +import java.util.*; + +import com.google.common.collect.SortedSetMultimap; +import com.google.common.collect.TreeMultimap; + +import org.apache.cassandra.config.CFMetaData; +import org.apache.cassandra.cql3.ColumnIdentifier; +import org.apache.cassandra.db.*; +import org.apache.cassandra.db.rows.Cell; +import org.apache.cassandra.db.rows.CellPath; +import org.apache.cassandra.config.ColumnDefinition; +import org.apache.cassandra.io.util.DataOutputPlus; +import org.apache.cassandra.utils.ByteBufferUtil; + +/** + * Represents which (non-PK) columns (and optionally which sub-part of a column for complex columns) are selected + * by a query. + * + * In practice, this class cover 2 main cases: + * 1) most user queries have to internally query all columns, because the CQL semantic requires us to know if + * a row is live or not even if it has no values for the columns requested by the user (see #6588for more + * details). However, while we need to know for columns if it has live values, we can actually save from + * sending the values for those columns that will not be returned to the user. + * 2) for some internal queries (and for queries using #6588 if we introduce it), we're actually fine only + * actually querying some of the columns. + * + * For complex columns, this class allows to be more fine grained than the column by only selection some of the + * cells of the complex column (either individual cell by path name, or some slice). + */ +public class ColumnFilter +{ + public static final Serializer serializer = new Serializer(); + + private static final Comparator keyComparator = new Comparator() + { + public int compare(ColumnIdentifier id1, ColumnIdentifier id2) + { + return ByteBufferUtil.compareUnsigned(id1.bytes, id2.bytes); + } + }; + private static final Comparator valueComparator = new Comparator() + { + public int compare(ColumnSubselection s1, ColumnSubselection s2) + { + assert s1.column().name.equals(s2.column().name); + return s1.column().cellPathComparator().compare(s1.minIncludedPath(), s2.minIncludedPath()); + } + }; + + // Distinguish between the 2 cases described above: if 'isFetchAll' is true, then all columns will be retrieved + // by the query, but the values for column/cells not selected by 'selection' and 'subSelections' will be skipped. + // Otherwise, only the column/cells returned by 'selection' and 'subSelections' will be returned at all. + private final boolean isFetchAll; + + private final CFMetaData metadata; // can be null if !isFetchAll + + private final PartitionColumns selection; // can be null if isFetchAll and we don't want to skip any value + private final SortedSetMultimap subSelections; // can be null + + private ColumnFilter(boolean isFetchAll, + CFMetaData metadata, + PartitionColumns columns, + SortedSetMultimap subSelections) + { + this.isFetchAll = isFetchAll; + this.metadata = metadata; + this.selection = columns; + this.subSelections = subSelections; + } + + /** + * A selection that includes all columns (and their values). + */ + public static ColumnFilter all(CFMetaData metadata) + { + return new ColumnFilter(true, metadata, null, null); + } + + /** + * A selection that only fetch the provided columns. + *

+ * Note that this shouldn't be used for CQL queries in general as all columns should be queried to + * preserve CQL semantic (see class javadoc). This is ok for some internal queries however (and + * for #6588 if/when we implement it). + */ + public static ColumnFilter selection(PartitionColumns columns) + { + return new ColumnFilter(false, null, columns, null); + } + + /** + * The columns that needs to be fetched internally for this selection. + *

+ * This is the columns that must be present in the internal rows returned by queries using this selection, + * not the columns that are actually queried by the user (see the class javadoc for details). + * + * @return the column to fetch for this selection. + */ + public PartitionColumns fetchedColumns() + { + return isFetchAll ? metadata.partitionColumns() : selection; + } + + /** + * Whether the provided column is selected by this selection. + */ + public boolean includes(ColumnDefinition column) + { + return isFetchAll || selection.contains(column); + } + + /** + * Whether we can skip the value for the provided selected column. + */ + public boolean canSkipValue(ColumnDefinition column) + { + return isFetchAll && selection != null && !selection.contains(column); + } + + /** + * Whether the provided cell of a complex column is selected by this selection. + */ + public boolean includes(Cell cell) + { + if (isFetchAll || subSelections == null || !cell.column().isComplex()) + return true; + + SortedSet s = subSelections.get(cell.column().name); + if (s.isEmpty()) + return true; + + for (ColumnSubselection subSel : s) + if (subSel.includes(cell.path())) + return true; + + return false; + } + + /** + * Whether we can skip the value of the cell of a complex column. + */ + public boolean canSkipValue(ColumnDefinition column, CellPath path) + { + if (!isFetchAll || subSelections == null || !column.isComplex()) + return false; + + SortedSet s = subSelections.get(column.name); + if (s.isEmpty()) + return false; + + for (ColumnSubselection subSel : s) + if (subSel.includes(path)) + return false; + + return true; + } + + /** + * Creates a new {@code Tester} to efficiently test the inclusion of cells of complex column + * {@code column}. + */ + public Tester newTester(ColumnDefinition column) + { + if (subSelections == null || !column.isComplex()) + return null; + + SortedSet s = subSelections.get(column.name); + if (s.isEmpty()) + return null; + + return new Tester(s.iterator()); + } + + /** + * Returns a {@code ColumnFilter}} builder that includes all columns (so the selections + * added to the builder are the columns/cells for which we shouldn't skip the values). + */ + public static Builder allColumnsBuilder(CFMetaData metadata) + { + return new Builder(metadata); + } + + /** + * Returns a {@code ColumnFilter}} builder that includes only the columns/cells + * added to the builder. + */ + public static Builder selectionBuilder() + { + return new Builder(null); + } + + public static class Tester + { + private ColumnSubselection current; + private final Iterator iterator; + + private Tester(Iterator iterator) + { + this.iterator = iterator; + } + + public boolean includes(CellPath path) + { + while (current == null) + { + if (!iterator.hasNext()) + return false; + + current = iterator.next(); + if (current.includes(path)) + return true; + + if (current.column().cellPathComparator().compare(current.maxIncludedPath(), path) < 0) + current = null; + } + return false; + } + + public boolean canSkipValue(CellPath path) + { + while (current == null) + { + if (!iterator.hasNext()) + return false; + + current = iterator.next(); + if (current.includes(path)) + return false; + + if (current.column().cellPathComparator().compare(current.maxIncludedPath(), path) < 0) + current = null; + } + return true; + } + } + + public static class Builder + { + private final CFMetaData metadata; + private PartitionColumns.Builder selection; + private List subSelections; + + public Builder(CFMetaData metadata) + { + this.metadata = metadata; + } + + public Builder add(ColumnDefinition c) + { + if (selection == null) + selection = PartitionColumns.builder(); + selection.add(c); + return this; + } + + public Builder addAll(Iterable columns) + { + if (selection == null) + selection = PartitionColumns.builder(); + selection.addAll(columns); + return this; + } + + private Builder addSubSelection(ColumnSubselection subSelection) + { + add(subSelection.column()); + if (subSelections == null) + subSelections = new ArrayList<>(); + subSelections.add(subSelection); + return this; + } + + public Builder slice(ColumnDefinition c, CellPath from, CellPath to) + { + return addSubSelection(ColumnSubselection.slice(c, from, to)); + } + + public Builder select(ColumnDefinition c, CellPath elt) + { + return addSubSelection(ColumnSubselection.element(c, elt)); + } + + public ColumnFilter build() + { + boolean isFetchAll = metadata != null; + assert isFetchAll || selection != null; + + SortedSetMultimap s = null; + if (subSelections != null) + { + s = TreeMultimap.create(keyComparator, valueComparator); + for (ColumnSubselection subSelection : subSelections) + s.put(subSelection.column().name, subSelection); + } + + return new ColumnFilter(isFetchAll, metadata, selection == null ? null : selection.build(), s); + } + } + + @Override + public String toString() + { + if (selection == null) + return "*"; + + Iterator defs = selection.selectOrderIterator(); + StringBuilder sb = new StringBuilder(); + appendColumnDef(sb, defs.next()); + while (defs.hasNext()) + appendColumnDef(sb.append(", "), defs.next()); + return sb.toString(); + } + + private void appendColumnDef(StringBuilder sb, ColumnDefinition column) + { + if (subSelections == null) + { + sb.append(column.name); + return; + } + + SortedSet s = subSelections.get(column.name); + if (s.isEmpty()) + { + sb.append(column.name); + return; + } + + int i = 0; + for (ColumnSubselection subSel : s) + sb.append(i++ == 0 ? "" : ", ").append(column.name).append(subSel); + } + + public static class Serializer + { + private static final int IS_FETCH_ALL_MASK = 0x01; + private static final int HAS_SELECTION_MASK = 0x02; + private static final int HAS_SUB_SELECTIONS_MASK = 0x04; + + private int makeHeaderByte(ColumnFilter selection) + { + return (selection.isFetchAll ? IS_FETCH_ALL_MASK : 0) + | (selection.selection != null ? HAS_SELECTION_MASK : 0) + | (selection.subSelections != null ? HAS_SUB_SELECTIONS_MASK : 0); + } + + public void serialize(ColumnFilter selection, DataOutputPlus out, int version) throws IOException + { + out.writeByte(makeHeaderByte(selection)); + + if (selection.selection != null) + { + Columns.serializer.serialize(selection.selection.statics, out); + Columns.serializer.serialize(selection.selection.regulars, out); + } + + if (selection.subSelections != null) + { + out.writeShort(selection.subSelections.size()); + for (ColumnSubselection subSel : selection.subSelections.values()) + ColumnSubselection.serializer.serialize(subSel, out, version); + } + } + + public ColumnFilter deserialize(DataInput in, int version, CFMetaData metadata) throws IOException + { + int header = in.readUnsignedByte(); + boolean isFetchAll = (header & IS_FETCH_ALL_MASK) != 0; + boolean hasSelection = (header & HAS_SELECTION_MASK) != 0; + boolean hasSubSelections = (header & HAS_SUB_SELECTIONS_MASK) != 0; + + PartitionColumns selection = null; + if (hasSelection) + { + Columns statics = Columns.serializer.deserialize(in, metadata); + Columns regulars = Columns.serializer.deserialize(in, metadata); + selection = new PartitionColumns(statics, regulars); + } + + SortedSetMultimap subSelections = null; + if (hasSubSelections) + { + subSelections = TreeMultimap.create(keyComparator, valueComparator); + int size = in.readUnsignedShort(); + for (int i = 0; i < size; i++) + { + ColumnSubselection subSel = ColumnSubselection.serializer.deserialize(in, version, metadata); + subSelections.put(subSel.column().name, subSel); + } + } + + return new ColumnFilter(isFetchAll, isFetchAll ? metadata : null, selection, subSelections); + } + + public long serializedSize(ColumnFilter selection, int version, TypeSizes sizes) + { + long size = 1; // header byte + + if (selection.selection != null) + { + size += Columns.serializer.serializedSize(selection.selection.statics, sizes); + size += Columns.serializer.serializedSize(selection.selection.regulars, sizes); + } + + if (selection.subSelections != null) + { + + size += sizes.sizeof((short)selection.subSelections.size()); + for (ColumnSubselection subSel : selection.subSelections.values()) + size += ColumnSubselection.serializer.serializedSize(subSel, version, sizes); + } + + return size; + } + } +} diff --git a/src/java/org/apache/cassandra/db/filter/ColumnSlice.java b/src/java/org/apache/cassandra/db/filter/ColumnSlice.java deleted file mode 100644 index 1cc348c8b6..0000000000 --- a/src/java/org/apache/cassandra/db/filter/ColumnSlice.java +++ /dev/null @@ -1,289 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.apache.cassandra.db.filter; - -import java.io.*; -import java.nio.ByteBuffer; -import java.util.ArrayList; -import java.util.Arrays; -import java.util.Comparator; -import java.util.List; - -import org.apache.cassandra.db.*; -import org.apache.cassandra.db.composites.*; -import org.apache.cassandra.db.marshal.AbstractType; -import org.apache.cassandra.io.ISerializer; -import org.apache.cassandra.io.IVersionedSerializer; -import org.apache.cassandra.io.util.DataOutputPlus; -import org.apache.cassandra.utils.ByteBufferUtil; - -public class ColumnSlice -{ - public static final ColumnSlice ALL_COLUMNS = new ColumnSlice(Composites.EMPTY, Composites.EMPTY); - public static final ColumnSlice[] ALL_COLUMNS_ARRAY = new ColumnSlice[]{ ALL_COLUMNS }; - - public final Composite start; - public final Composite finish; - - public ColumnSlice(Composite start, Composite finish) - { - assert start != null && finish != null; - this.start = start; - this.finish = finish; - } - - public boolean isAlwaysEmpty(CellNameType comparator, boolean reversed) - { - Comparator orderedComparator = reversed ? comparator.reverseComparator() : comparator; - return !start.isEmpty() && !finish.isEmpty() && orderedComparator.compare(start, finish) > 0; - } - - public boolean includes(Comparator cmp, Composite name) - { - return (start.isEmpty() || cmp.compare(start, name) <= 0) && (finish.isEmpty() || cmp.compare(finish, name) >= 0); - } - - public boolean isBefore(Comparator cmp, Composite name) - { - return !finish.isEmpty() && cmp.compare(finish, name) < 0; - } - - public boolean intersects(List minCellNames, List maxCellNames, CellNameType comparator, boolean reversed) - { - Composite sStart = reversed ? finish : start; - Composite sEnd = reversed ? start : finish; - - if (compare(sStart, maxCellNames, comparator, true) > 0 || compare(sEnd, minCellNames, comparator, false) < 0) - return false; - - // We could safely return true here, but there's a minor optimization: if the first component is restricted - // to a single value, we can check that the second component falls within the min/max for that component - // (and repeat for all components). - for (int i = 0; i < minCellNames.size() && i < maxCellNames.size(); i++) - { - AbstractType t = comparator.subtype(i); - ByteBuffer s = i < sStart.size() ? sStart.get(i) : ByteBufferUtil.EMPTY_BYTE_BUFFER; - ByteBuffer f = i < sEnd.size() ? sEnd.get(i) : ByteBufferUtil.EMPTY_BYTE_BUFFER; - - // we already know the first component falls within its min/max range (otherwise we wouldn't get here) - if (i > 0 && (i < sEnd.size() && t.compare(f, minCellNames.get(i)) < 0 || - i < sStart.size() && t.compare(s, maxCellNames.get(i)) > 0)) - return false; - - // if this component isn't equal in the start and finish, we don't need to check any more - if (i >= sStart.size() || i >= sEnd.size() || t.compare(s, f) != 0) - break; - } - - return true; - } - - /** Helper method for intersects() */ - private int compare(Composite sliceBounds, List sstableBounds, CellNameType comparator, boolean isSliceStart) - { - for (int i = 0; i < sstableBounds.size(); i++) - { - if (i >= sliceBounds.size()) - { - // When isSliceStart is true, we're comparing the end of the slice against the min cell name for the sstable, - // so the slice is something like [(1, 0), (1, 0)], and the sstable max is something like (1, 0, 1). - // We want to return -1 (slice start is smaller than max column name) so that we say the slice intersects. - // The opposite is true when dealing with the end slice. For example, with the same slice and a min - // cell name of (1, 0, 1), we want to return 1 (slice end is bigger than min column name). - return isSliceStart ? -1 : 1; - } - - int comparison = comparator.subtype(i).compare(sliceBounds.get(i), sstableBounds.get(i)); - if (comparison != 0) - return comparison; - } - - // the slice bound and sstable bound have been equal in all components so far - if (sliceBounds.size() > sstableBounds.size()) - { - // We have the opposite situation from the one described above. With a slice of [(1, 0), (1, 0)], - // and a min/max cell name of (1), we want to say the slice start is smaller than the max and the slice - // end is larger than the min. - return isSliceStart ? -1 : 1; - } - - return 0; - } - - /** - * Validates that the provided slice array contains only non-overlapped slices valid for a query {@code reversed} - * or not on a table using {@code comparator}. - */ - public static boolean validateSlices(ColumnSlice[] slices, CellNameType type, boolean reversed) - { - Comparator comparator = reversed ? type.reverseComparator() : type; - - for (int i = 0; i < slices.length; i++) - { - Composite start = slices[i].start; - Composite finish = slices[i].finish; - - if (start.isEmpty() || finish.isEmpty()) - { - if (start.isEmpty() && i > 0) - return false; - - if (finish.isEmpty()) - return i == slices.length - 1; - } - else - { - // !finish.isEmpty() is imposed by prior loop - if (i > 0 && comparator.compare(slices[i - 1].finish, start) >= 0) - return false; - - if (comparator.compare(start, finish) > 0) - return false; - } - } - return true; - } - - /** - * Takes an array of slices (potentially overlapping and in any order, though each individual slice must have - * its start before or equal its end in {@code comparator} orde) and return an equivalent array of non-overlapping - * slices in {@code comparator order}. - * - * @param slices an array of slices. This may be modified by this method. - * @param comparator the order in which to sort the slices. - * @return the smallest possible array of non-overlapping slices in {@code compator} order. If the original - * slices are already non-overlapping and in comparator order, this may or may not return the provided slices - * directly. - */ - public static ColumnSlice[] deoverlapSlices(ColumnSlice[] slices, final Comparator comparator) - { - if (slices.length <= 1) - return slices; - - Arrays.sort(slices, new Comparator() - { - @Override - public int compare(ColumnSlice s1, ColumnSlice s2) - { - if (s1.start.isEmpty() || s2.start.isEmpty()) - { - if (s1.start.isEmpty() != s2.start.isEmpty()) - return s1.start.isEmpty() ? -1 : 1; - } - else - { - int c = comparator.compare(s1.start, s2.start); - if (c != 0) - return c; - } - - // For the finish, empty always means greater - return s1.finish.isEmpty() || s2.finish.isEmpty() - ? (s1.finish.isEmpty() ? 1 : -1) - : comparator.compare(s1.finish, s2.finish); - } - }); - - List slicesCopy = new ArrayList<>(slices.length); - - ColumnSlice last = slices[0]; - - for (int i = 1; i < slices.length; i++) - { - ColumnSlice s2 = slices[i]; - - boolean includesStart = last.includes(comparator, s2.start); - boolean includesFinish = s2.finish.isEmpty() ? last.finish.isEmpty() : last.includes(comparator, s2.finish); - - if (includesStart && includesFinish) - continue; - - if (!includesStart && !includesFinish) - { - slicesCopy.add(last); - last = s2; - continue; - } - - if (includesStart) - { - last = new ColumnSlice(last.start, s2.finish); - continue; - } - - assert !includesFinish; - } - - slicesCopy.add(last); - - return slicesCopy.toArray(new ColumnSlice[slicesCopy.size()]); - } - - @Override - public final int hashCode() - { - int hashCode = 31 + start.hashCode(); - return 31*hashCode + finish.hashCode(); - } - - @Override - public final boolean equals(Object o) - { - if(!(o instanceof ColumnSlice)) - return false; - ColumnSlice that = (ColumnSlice)o; - return start.equals(that.start) && finish.equals(that.finish); - } - - @Override - public String toString() - { - return "[" + ByteBufferUtil.bytesToHex(start.toByteBuffer()) + ", " + ByteBufferUtil.bytesToHex(finish.toByteBuffer()) + "]"; - } - - public static class Serializer implements IVersionedSerializer - { - private final CType type; - - public Serializer(CType type) - { - this.type = type; - } - - public void serialize(ColumnSlice cs, DataOutputPlus out, int version) throws IOException - { - ISerializer serializer = type.serializer(); - serializer.serialize(cs.start, out); - serializer.serialize(cs.finish, out); - } - - public ColumnSlice deserialize(DataInput in, int version) throws IOException - { - ISerializer serializer = type.serializer(); - Composite start = serializer.deserialize(in); - Composite finish = serializer.deserialize(in); - return new ColumnSlice(start, finish); - } - - public long serializedSize(ColumnSlice cs, int version) - { - ISerializer serializer = type.serializer(); - return serializer.serializedSize(cs.start, TypeSizes.NATIVE) + serializer.serializedSize(cs.finish, TypeSizes.NATIVE); - } - } -} diff --git a/src/java/org/apache/cassandra/db/filter/ColumnSubselection.java b/src/java/org/apache/cassandra/db/filter/ColumnSubselection.java new file mode 100644 index 0000000000..35db6f26ff --- /dev/null +++ b/src/java/org/apache/cassandra/db/filter/ColumnSubselection.java @@ -0,0 +1,233 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.cassandra.db.filter; + +import java.io.DataInput; +import java.io.IOException; +import java.nio.ByteBuffer; +import java.util.Comparator; + +import org.apache.cassandra.config.CFMetaData; +import org.apache.cassandra.config.ColumnDefinition; +import org.apache.cassandra.db.TypeSizes; +import org.apache.cassandra.db.rows.CellPath; +import org.apache.cassandra.db.marshal.AbstractType; +import org.apache.cassandra.db.marshal.CollectionType; +import org.apache.cassandra.db.marshal.UTF8Type; +import org.apache.cassandra.io.util.DataOutputPlus; +import org.apache.cassandra.utils.ByteBufferUtil; + +/** + * Handles the selection of a subpart of a column. + *

+ * This only make sense for complex column. For those, this allow for instance + * to select only a slice of a map. + */ +public abstract class ColumnSubselection +{ + public static final Serializer serializer = new Serializer(); + + private enum Kind { SLICE, ELEMENT } + + protected final ColumnDefinition column; + + protected ColumnSubselection(ColumnDefinition column) + { + this.column = column; + } + + public static ColumnSubselection slice(ColumnDefinition column, CellPath from, CellPath to) + { + assert column.isComplex() && column.type instanceof CollectionType; + assert from.size() <= 1 && to.size() <= 1; + return new Slice(column, from, to); + } + + public static ColumnSubselection element(ColumnDefinition column, CellPath elt) + { + assert column.isComplex() && column.type instanceof CollectionType; + assert elt.size() == 1; + return new Element(column, elt); + } + + public ColumnDefinition column() + { + return column; + } + + protected abstract Kind kind(); + + public abstract CellPath minIncludedPath(); + public abstract CellPath maxIncludedPath(); + public abstract boolean includes(CellPath path); + + private static class Slice extends ColumnSubselection + { + private final CellPath from; + private final CellPath to; + + private Slice(ColumnDefinition column, CellPath from, CellPath to) + { + super(column); + this.from = from; + this.to = to; + } + + protected Kind kind() + { + return Kind.SLICE; + } + + public CellPath minIncludedPath() + { + return from; + } + + public CellPath maxIncludedPath() + { + return to; + } + + public boolean includes(CellPath path) + { + Comparator cmp = column.cellPathComparator(); + return cmp.compare(from, path) <= 0 && cmp.compare(path, to) <= 0; + } + + @Override + public String toString() + { + // This assert we're dealing with a collection since that's the only thing it's used for so far. + AbstractType type = ((CollectionType)column().type).nameComparator(); + return String.format("[%s:%s]", from == CellPath.BOTTOM ? "" : type.getString(from.get(0)), to == CellPath.TOP ? "" : type.getString(to.get(0))); + } + } + + private static class Element extends ColumnSubselection + { + private final CellPath element; + + private Element(ColumnDefinition column, CellPath elt) + { + super(column); + this.element = elt; + } + + protected Kind kind() + { + return Kind.ELEMENT; + } + + public CellPath minIncludedPath() + { + return element; + } + + public CellPath maxIncludedPath() + { + return element; + } + + public boolean includes(CellPath path) + { + Comparator cmp = column.cellPathComparator(); + return cmp.compare(element, path) == 0; + } + + @Override + public String toString() + { + // This assert we're dealing with a collection since that's the only thing it's used for so far. + AbstractType type = ((CollectionType)column().type).nameComparator(); + return String.format("[%s]", type.getString(element.get(0))); + } + } + + public static class Serializer + { + public void serialize(ColumnSubselection subSel, DataOutputPlus out, int version) throws IOException + { + ColumnDefinition column = subSel.column(); + ByteBufferUtil.writeWithShortLength(column.name.bytes, out); + out.writeByte(subSel.kind().ordinal()); + switch (subSel.kind()) + { + case SLICE: + Slice slice = (Slice)subSel; + column.cellPathSerializer().serialize(slice.from, out); + column.cellPathSerializer().serialize(slice.to, out); + break; + case ELEMENT: + Element eltSelection = (Element)subSel; + column.cellPathSerializer().serialize(eltSelection.element, out); + break; + } + throw new AssertionError(); + } + + public ColumnSubselection deserialize(DataInput in, int version, CFMetaData metadata) throws IOException + { + ByteBuffer name = ByteBufferUtil.readWithShortLength(in); + ColumnDefinition column = metadata.getColumnDefinition(name); + if (column == null) + { + // If we don't find the definition, it could be we have data for a dropped column, and we shouldn't + // fail deserialization because of that. So we grab a "fake" ColumnDefinition that ensure proper + // deserialization. The column will be ignore later on anyway. + column = metadata.getDroppedColumnDefinition(name); + if (column == null) + throw new RuntimeException("Unknown column " + UTF8Type.instance.getString(name) + " during deserialization"); + } + + Kind kind = Kind.values()[in.readUnsignedByte()]; + switch (kind) + { + case SLICE: + CellPath from = column.cellPathSerializer().deserialize(in); + CellPath to = column.cellPathSerializer().deserialize(in); + return new Slice(column, from, to); + case ELEMENT: + CellPath elt = column.cellPathSerializer().deserialize(in); + return new Element(column, elt); + } + throw new AssertionError(); + } + + public long serializedSize(ColumnSubselection subSel, int version, TypeSizes sizes) + { + long size = 0; + + ColumnDefinition column = subSel.column(); + size += sizes.sizeofWithShortLength(column.name.bytes); + size += 1; // kind + switch (subSel.kind()) + { + case SLICE: + Slice slice = (Slice)subSel; + size += column.cellPathSerializer().serializedSize(slice.from, sizes); + size += column.cellPathSerializer().serializedSize(slice.to, sizes); + break; + case ELEMENT: + Element element = (Element)subSel; + size += column.cellPathSerializer().serializedSize(element.element, sizes); + break; + } + return size; + } + } +} diff --git a/src/java/org/apache/cassandra/db/filter/DataLimits.java b/src/java/org/apache/cassandra/db/filter/DataLimits.java new file mode 100644 index 0000000000..42bfa4e649 --- /dev/null +++ b/src/java/org/apache/cassandra/db/filter/DataLimits.java @@ -0,0 +1,737 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.cassandra.db.filter; + +import java.io.DataInput; +import java.io.IOException; +import java.nio.ByteBuffer; + +import org.apache.cassandra.db.*; +import org.apache.cassandra.db.rows.*; +import org.apache.cassandra.db.partitions.*; +import org.apache.cassandra.io.util.DataOutputPlus; +import org.apache.cassandra.utils.ByteBufferUtil; + +/** + * Object in charge of tracking if we have fetch enough data for a given query. + * + * The reason this is not just a simple integer is that Thrift and CQL3 count + * stuffs in different ways. This is what abstract those differences. + */ +public abstract class DataLimits +{ + public static final Serializer serializer = new Serializer(); + + public static final DataLimits NONE = new CQLLimits(Integer.MAX_VALUE) + { + @Override + public boolean hasEnoughLiveData(CachedPartition cached, int nowInSec) + { + return false; + } + + @Override + public UnfilteredPartitionIterator filter(UnfilteredPartitionIterator iter, int nowInSec) + { + return iter; + } + + @Override + public UnfilteredRowIterator filter(UnfilteredRowIterator iter, int nowInSec) + { + return iter; + } + }; + + // We currently deal with distinct queries by querying full partitions but limiting the result at 1 row per + // partition (see SelectStatement.makeFilter). So an "unbounded" distinct is still actually doing some filtering. + public static final DataLimits DISTINCT_NONE = new CQLLimits(Integer.MAX_VALUE, 1, true); + + private enum Kind { CQL_LIMIT, CQL_PAGING_LIMIT, THRIFT_LIMIT, SUPER_COLUMN_COUNTING_LIMIT } + + public static DataLimits cqlLimits(int cqlRowLimit) + { + return new CQLLimits(cqlRowLimit); + } + + public static DataLimits cqlLimits(int cqlRowLimit, int perPartitionLimit) + { + return new CQLLimits(cqlRowLimit, perPartitionLimit); + } + + public static DataLimits distinctLimits(int cqlRowLimit) + { + return CQLLimits.distinct(cqlRowLimit); + } + + public static DataLimits thriftLimits(int partitionLimit, int cellPerPartitionLimit) + { + return new ThriftLimits(partitionLimit, cellPerPartitionLimit); + } + + public static DataLimits superColumnCountingLimits(int partitionLimit, int cellPerPartitionLimit) + { + return new SuperColumnCountingLimits(partitionLimit, cellPerPartitionLimit); + } + + protected abstract Kind kind(); + + public abstract boolean isUnlimited(); + + public abstract DataLimits forPaging(int pageSize); + public abstract DataLimits forPaging(int pageSize, ByteBuffer lastReturnedKey, int lastReturnedKeyRemaining); + + public abstract DataLimits forShortReadRetry(int toFetch); + + public abstract boolean hasEnoughLiveData(CachedPartition cached, int nowInSec); + + /** + * Returns a new {@code Counter} for this limits. + * + * @param nowInSec the current time in second (to decide what is expired or not). + * @param assumeLiveData if true, the counter will assume that every row passed is live and won't + * thus check for liveness, otherwise it will. This should be {@code true} when used on a + * {@code RowIterator} (since it only returns live rows), false otherwise. + * @return a new {@code Counter} for this limits. + */ + public abstract Counter newCounter(int nowInSec, boolean assumeLiveData); + + /** + * The max number of results this limits enforces. + *

+ * Note that the actual definition of "results" depends a bit: for CQL, it's always rows, but for + * thrift, it means cells. The {@link #countsCells} allows to distinguish between the two cases if + * needed. + * + * @return the maximum number of results this limits enforces. + */ + public abstract int count(); + + public abstract int perPartitionCount(); + + public abstract boolean countsCells(); + + public UnfilteredPartitionIterator filter(UnfilteredPartitionIterator iter, int nowInSec) + { + return new CountingUnfilteredPartitionIterator(iter, newCounter(nowInSec, false)); + } + + public UnfilteredRowIterator filter(UnfilteredRowIterator iter, int nowInSec) + { + return new CountingUnfilteredRowIterator(iter, newCounter(nowInSec, false)); + } + + public PartitionIterator filter(PartitionIterator iter, int nowInSec) + { + return new CountingPartitionIterator(iter, this, nowInSec); + } + + /** + * Estimate the number of results (the definition of "results" will be rows for CQL queries + * and partitions for thrift ones) that a full scan of the provided cfs would yield. + */ + public abstract float estimateTotalResults(ColumnFamilyStore cfs); + + public interface Counter + { + public void newPartition(DecoratedKey partitionKey, Row staticRow); + public void newRow(Row row); + public void endOfPartition(); + + /** + * The number of results counted. + *

+ * Note that the definition of "results" should be the same that for {@link #count}. + * + * @return the number of results counted. + */ + public int counted(); + + public int countedInCurrentPartition(); + + public boolean isDone(); + public boolean isDoneForPartition(); + } + + /** + * Limits used by CQL; this counts rows. + */ + private static class CQLLimits extends DataLimits + { + protected final int rowLimit; + protected final int perPartitionLimit; + + // Whether the query is a distinct query or not. This is currently not used by the code but prior experience + // shows that keeping the information around is wise and might be useful in the future. + protected final boolean isDistinct; + + private CQLLimits(int rowLimit) + { + this(rowLimit, Integer.MAX_VALUE); + } + + private CQLLimits(int rowLimit, int perPartitionLimit) + { + this(rowLimit, perPartitionLimit, false); + } + + private CQLLimits(int rowLimit, int perPartitionLimit, boolean isDistinct) + { + this.rowLimit = rowLimit; + this.perPartitionLimit = perPartitionLimit; + this.isDistinct = isDistinct; + } + + private static CQLLimits distinct(int rowLimit) + { + return new CQLLimits(rowLimit, 1, true); + } + + protected Kind kind() + { + return Kind.CQL_LIMIT; + } + + public boolean isUnlimited() + { + return rowLimit == Integer.MAX_VALUE && perPartitionLimit == Integer.MAX_VALUE; + } + + public DataLimits forPaging(int pageSize) + { + return new CQLLimits(pageSize, perPartitionLimit); + } + + public DataLimits forPaging(int pageSize, ByteBuffer lastReturnedKey, int lastReturnedKeyRemaining) + { + return new CQLPagingLimits(pageSize, perPartitionLimit, isDistinct, lastReturnedKey, lastReturnedKeyRemaining); + } + + public DataLimits forShortReadRetry(int toFetch) + { + // When we do a short read retry, we're only ever querying the single partition on which we have a short read. So + // we use toFetch as the row limit and use no perPartitionLimit (it would be equivalent in practice to use toFetch + // for both argument or just for perPartitionLimit with no limit on rowLimit). + return new CQLLimits(toFetch, Integer.MAX_VALUE, isDistinct); + } + + public boolean hasEnoughLiveData(CachedPartition cached, int nowInSec) + { + // We want the number of row that are currently live. Getting that precise number forces + // us to iterate the cached partition in general, but we can avoid that if: + // - The number of rows with at least one non-expiring cell is greater than what we ask, + // in which case we know we have enough live. + // - The number of rows is less than requested, in which case we know we won't have enough. + if (cached.rowsWithNonExpiringCells() >= rowLimit) + return true; + + if (cached.rowCount() < rowLimit) + return false; + + // Otherwise, we need to re-count + try (UnfilteredRowIterator cacheIter = cached.unfilteredIterator(ColumnFilter.selection(cached.columns()), Slices.ALL, false); + CountingUnfilteredRowIterator iter = new CountingUnfilteredRowIterator(cacheIter, newCounter(nowInSec, false))) + { + // Consume the iterator until we've counted enough + while (iter.hasNext() && !iter.counter().isDone()) + iter.next(); + return iter.counter().isDone(); + } + } + + public Counter newCounter(int nowInSec, boolean assumeLiveData) + { + return new CQLCounter(nowInSec, assumeLiveData); + } + + public int count() + { + return rowLimit; + } + + public int perPartitionCount() + { + return perPartitionLimit; + } + + public boolean countsCells() + { + return false; + } + + public float estimateTotalResults(ColumnFamilyStore cfs) + { + // TODO: we should start storing stats on the number of rows (instead of the number of cells, which + // is what getMeanColumns returns) + float rowsPerPartition = ((float) cfs.getMeanColumns()) / cfs.metadata.partitionColumns().regulars.columnCount(); + return rowsPerPartition * (cfs.estimateKeys()); + } + + protected class CQLCounter implements Counter + { + protected final int nowInSec; + protected final boolean assumeLiveData; + + protected int rowCounted; + protected int rowInCurrentPartition; + + protected boolean hasLiveStaticRow; + + public CQLCounter(int nowInSec, boolean assumeLiveData) + { + this.nowInSec = nowInSec; + this.assumeLiveData = assumeLiveData; + } + + public void newPartition(DecoratedKey partitionKey, Row staticRow) + { + rowInCurrentPartition = 0; + if (!staticRow.isEmpty() && (assumeLiveData || staticRow.hasLiveData(nowInSec))) + hasLiveStaticRow = true; + } + + public void endOfPartition() + { + // Normally, we don't count static rows as from a CQL point of view, it will be merge with other + // rows in the partition. However, if we only have the static row, it will be returned as one row + // so count it. + if (hasLiveStaticRow && rowInCurrentPartition == 0) + ++rowCounted; + } + + public int counted() + { + return rowCounted; + } + + public int countedInCurrentPartition() + { + return rowInCurrentPartition; + } + + public boolean isDone() + { + return rowCounted >= rowLimit; + } + + public boolean isDoneForPartition() + { + return isDone() || rowInCurrentPartition >= perPartitionLimit; + } + + public void newRow(Row row) + { + if (assumeLiveData || row.hasLiveData(nowInSec)) + { + ++rowCounted; + ++rowInCurrentPartition; + } + } + } + + @Override + public String toString() + { + StringBuilder sb = new StringBuilder(); + + if (rowLimit != Integer.MAX_VALUE) + { + sb.append("LIMIT ").append(rowLimit); + if (perPartitionLimit != Integer.MAX_VALUE) + sb.append(" "); + } + + if (perPartitionLimit != Integer.MAX_VALUE) + sb.append("PER PARTITION LIMIT ").append(perPartitionLimit); + + return sb.toString(); + } + } + + private static class CQLPagingLimits extends CQLLimits + { + private final ByteBuffer lastReturnedKey; + private final int lastReturnedKeyRemaining; + + public CQLPagingLimits(int rowLimit, int perPartitionLimit, boolean isDistinct, ByteBuffer lastReturnedKey, int lastReturnedKeyRemaining) + { + super(rowLimit, perPartitionLimit, isDistinct); + this.lastReturnedKey = lastReturnedKey; + this.lastReturnedKeyRemaining = lastReturnedKeyRemaining; + } + + @Override + protected Kind kind() + { + return Kind.CQL_PAGING_LIMIT; + } + + @Override + public DataLimits forPaging(int pageSize) + { + throw new UnsupportedOperationException(); + } + + @Override + public DataLimits forPaging(int pageSize, ByteBuffer lastReturnedKey, int lastReturnedKeyRemaining) + { + throw new UnsupportedOperationException(); + } + + @Override + public Counter newCounter(int nowInSec, boolean assumeLiveData) + { + return new PagingAwareCounter(nowInSec, assumeLiveData); + } + + private class PagingAwareCounter extends CQLCounter + { + private PagingAwareCounter(int nowInSec, boolean assumeLiveData) + { + super(nowInSec, assumeLiveData); + } + + @Override + public void newPartition(DecoratedKey partitionKey, Row staticRow) + { + if (partitionKey.getKey().equals(lastReturnedKey)) + { + rowInCurrentPartition = perPartitionLimit - lastReturnedKeyRemaining; + // lastReturnedKey is the last key for which we're returned rows in the first page. + // So, since we know we have returned rows, we know we have accounted for the static row + // if any already, so force hasLiveStaticRow to false so we make sure to not count it + // once more. + hasLiveStaticRow = false; + } + else + { + super.newPartition(partitionKey, staticRow); + } + } + } + } + + /** + * Limits used by thrift; this count partition and cells. + */ + private static class ThriftLimits extends DataLimits + { + protected final int partitionLimit; + protected final int cellPerPartitionLimit; + + private ThriftLimits(int partitionLimit, int cellPerPartitionLimit) + { + this.partitionLimit = partitionLimit; + this.cellPerPartitionLimit = cellPerPartitionLimit; + } + + protected Kind kind() + { + return Kind.THRIFT_LIMIT; + } + + public boolean isUnlimited() + { + return partitionLimit == Integer.MAX_VALUE && cellPerPartitionLimit == Integer.MAX_VALUE; + } + + public DataLimits forPaging(int pageSize) + { + // We don't support paging on thrift in general but do use paging under the hood for get_count. For + // that case, we only care about limiting cellPerPartitionLimit (since it's paging over a single + // partition). We do check that the partition limit is 1 however to make sure this is not misused + // (as this wouldn't work properly for range queries). + assert partitionLimit == 1; + return new ThriftLimits(partitionLimit, pageSize); + } + + public DataLimits forPaging(int pageSize, ByteBuffer lastReturnedKey, int lastReturnedKeyRemaining) + { + throw new UnsupportedOperationException(); + } + + public DataLimits forShortReadRetry(int toFetch) + { + // Short read retries are always done for a single partition at a time, so it's ok to ignore the + // partition limit for those + return new ThriftLimits(1, toFetch); + } + + public boolean hasEnoughLiveData(CachedPartition cached, int nowInSec) + { + // We want the number of cells that are currently live. Getting that precise number forces + // us to iterate the cached partition in general, but we can avoid that if: + // - The number of non-expiring live cells is greater than the number of cells asked (we then + // know we have enough live cells). + // - The number of cells cached is less than requested, in which case we know we won't have enough. + if (cached.nonExpiringLiveCells() >= cellPerPartitionLimit) + return true; + + if (cached.nonTombstoneCellCount() < cellPerPartitionLimit) + return false; + + // Otherwise, we need to re-count + try (UnfilteredRowIterator cacheIter = cached.unfilteredIterator(ColumnFilter.selection(cached.columns()), Slices.ALL, false); + CountingUnfilteredRowIterator iter = new CountingUnfilteredRowIterator(cacheIter, newCounter(nowInSec, false))) + { + // Consume the iterator until we've counted enough + while (iter.hasNext() && !iter.counter().isDone()) + iter.next(); + return iter.counter().isDone(); + } + } + + public Counter newCounter(int nowInSec, boolean assumeLiveData) + { + return new ThriftCounter(nowInSec, assumeLiveData); + } + + public int count() + { + return partitionLimit * cellPerPartitionLimit; + } + + public int perPartitionCount() + { + return cellPerPartitionLimit; + } + + public boolean countsCells() + { + return true; + } + + public float estimateTotalResults(ColumnFamilyStore cfs) + { + // remember that getMeansColumns returns a number of cells: we should clean nomenclature + float cellsPerPartition = ((float) cfs.getMeanColumns()) / cfs.metadata.partitionColumns().regulars.columnCount(); + return cellsPerPartition * cfs.estimateKeys(); + } + + protected class ThriftCounter implements Counter + { + protected final int nowInSec; + protected final boolean assumeLiveData; + + protected int partitionsCounted; + protected int cellsCounted; + protected int cellsInCurrentPartition; + + public ThriftCounter(int nowInSec, boolean assumeLiveData) + { + this.nowInSec = nowInSec; + this.assumeLiveData = assumeLiveData; + } + + public void newPartition(DecoratedKey partitionKey, Row staticRow) + { + cellsInCurrentPartition = 0; + if (!staticRow.isEmpty()) + newRow(staticRow); + } + + public void endOfPartition() + { + ++partitionsCounted; + } + + public int counted() + { + return cellsCounted; + } + + public int countedInCurrentPartition() + { + return cellsInCurrentPartition; + } + + public boolean isDone() + { + return partitionsCounted >= partitionLimit; + } + + public boolean isDoneForPartition() + { + return isDone() || cellsInCurrentPartition >= cellPerPartitionLimit; + } + + public void newRow(Row row) + { + for (Cell cell : row) + { + if (assumeLiveData || cell.isLive(nowInSec)) + { + ++cellsCounted; + ++cellsInCurrentPartition; + } + } + } + } + + @Override + public String toString() + { + // This is not valid CQL, but that's ok since it's not used for CQL queries. + return String.format("THRIFT LIMIT (partitions=%d, cells_per_partition=%d)", partitionLimit, cellPerPartitionLimit); + } + } + + /** + * Limits used for thrift get_count when we only want to count super columns. + */ + private static class SuperColumnCountingLimits extends ThriftLimits + { + private SuperColumnCountingLimits(int partitionLimit, int cellPerPartitionLimit) + { + super(partitionLimit, cellPerPartitionLimit); + } + + protected Kind kind() + { + return Kind.SUPER_COLUMN_COUNTING_LIMIT; + } + + public DataLimits forPaging(int pageSize) + { + // We don't support paging on thrift in general but do use paging under the hood for get_count. For + // that case, we only care about limiting cellPerPartitionLimit (since it's paging over a single + // partition). We do check that the partition limit is 1 however to make sure this is not misused + // (as this wouldn't work properly for range queries). + assert partitionLimit == 1; + return new SuperColumnCountingLimits(partitionLimit, pageSize); + } + + public DataLimits forShortReadRetry(int toFetch) + { + // Short read retries are always done for a single partition at a time, so it's ok to ignore the + // partition limit for those + return new SuperColumnCountingLimits(1, toFetch); + } + + public Counter newCounter(int nowInSec, boolean assumeLiveData) + { + return new SuperColumnCountingCounter(nowInSec, assumeLiveData); + } + + protected class SuperColumnCountingCounter extends ThriftCounter + { + public SuperColumnCountingCounter(int nowInSec, boolean assumeLiveData) + { + super(nowInSec, assumeLiveData); + } + + public void newRow(Row row) + { + // In the internal format, a row == a super column, so that's what we want to count. + if (assumeLiveData || row.hasLiveData(nowInSec)) + { + ++cellsCounted; + ++cellsInCurrentPartition; + } + } + } + } + + public static class Serializer + { + public void serialize(DataLimits limits, DataOutputPlus out, int version) throws IOException + { + out.writeByte(limits.kind().ordinal()); + switch (limits.kind()) + { + case CQL_LIMIT: + case CQL_PAGING_LIMIT: + CQLLimits cqlLimits = (CQLLimits)limits; + out.writeInt(cqlLimits.rowLimit); + out.writeInt(cqlLimits.perPartitionLimit); + out.writeBoolean(cqlLimits.isDistinct); + if (limits.kind() == Kind.CQL_PAGING_LIMIT) + { + CQLPagingLimits pagingLimits = (CQLPagingLimits)cqlLimits; + ByteBufferUtil.writeWithShortLength(pagingLimits.lastReturnedKey, out); + out.writeInt(pagingLimits.lastReturnedKeyRemaining); + } + break; + case THRIFT_LIMIT: + case SUPER_COLUMN_COUNTING_LIMIT: + ThriftLimits thriftLimits = (ThriftLimits)limits; + out.writeInt(thriftLimits.partitionLimit); + out.writeInt(thriftLimits.cellPerPartitionLimit); + break; + } + } + + public DataLimits deserialize(DataInput in, int version) throws IOException + { + Kind kind = Kind.values()[in.readUnsignedByte()]; + switch (kind) + { + case CQL_LIMIT: + case CQL_PAGING_LIMIT: + int rowLimit = in.readInt(); + int perPartitionLimit = in.readInt(); + boolean isDistinct = in.readBoolean(); + if (kind == Kind.CQL_LIMIT) + return new CQLLimits(rowLimit, perPartitionLimit, isDistinct); + + ByteBuffer lastKey = ByteBufferUtil.readWithShortLength(in); + int lastRemaining = in.readInt(); + return new CQLPagingLimits(rowLimit, perPartitionLimit, isDistinct, lastKey, lastRemaining); + case THRIFT_LIMIT: + case SUPER_COLUMN_COUNTING_LIMIT: + int partitionLimit = in.readInt(); + int cellPerPartitionLimit = in.readInt(); + return kind == Kind.THRIFT_LIMIT + ? new ThriftLimits(partitionLimit, cellPerPartitionLimit) + : new SuperColumnCountingLimits(partitionLimit, cellPerPartitionLimit); + } + throw new AssertionError(); + } + + public long serializedSize(DataLimits limits, int version) + { + TypeSizes sizes = TypeSizes.NATIVE; + long size = sizes.sizeof((byte)limits.kind().ordinal()); + switch (limits.kind()) + { + case CQL_LIMIT: + case CQL_PAGING_LIMIT: + CQLLimits cqlLimits = (CQLLimits)limits; + size += sizes.sizeof(cqlLimits.rowLimit); + size += sizes.sizeof(cqlLimits.perPartitionLimit); + size += sizes.sizeof(cqlLimits.isDistinct); + if (limits.kind() == Kind.CQL_PAGING_LIMIT) + { + CQLPagingLimits pagingLimits = (CQLPagingLimits)cqlLimits; + size += ByteBufferUtil.serializedSizeWithShortLength(pagingLimits.lastReturnedKey, sizes); + size += sizes.sizeof(pagingLimits.lastReturnedKeyRemaining); + } + break; + case THRIFT_LIMIT: + case SUPER_COLUMN_COUNTING_LIMIT: + ThriftLimits thriftLimits = (ThriftLimits)limits; + size += sizes.sizeof(thriftLimits.partitionLimit); + size += sizes.sizeof(thriftLimits.cellPerPartitionLimit); + break; + default: + throw new AssertionError(); + } + return size; + } + } +} diff --git a/src/java/org/apache/cassandra/db/filter/ExtendedFilter.java b/src/java/org/apache/cassandra/db/filter/ExtendedFilter.java deleted file mode 100644 index 50ab57dfb2..0000000000 --- a/src/java/org/apache/cassandra/db/filter/ExtendedFilter.java +++ /dev/null @@ -1,499 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.apache.cassandra.db.filter; - -import java.nio.ByteBuffer; -import java.util.Collections; -import java.util.Iterator; -import java.util.List; -import java.util.SortedSet; -import java.util.TreeSet; - -import com.google.common.base.Objects; - -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; - -import org.apache.cassandra.config.ColumnDefinition; -import org.apache.cassandra.config.DatabaseDescriptor; -import org.apache.cassandra.cql3.Operator; -import org.apache.cassandra.db.Cell; -import org.apache.cassandra.db.ColumnFamily; -import org.apache.cassandra.db.ColumnFamilyStore; -import org.apache.cassandra.db.DataRange; -import org.apache.cassandra.db.DecoratedKey; -import org.apache.cassandra.db.IndexExpression; -import org.apache.cassandra.db.composites.CellName; -import org.apache.cassandra.db.composites.Composite; -import org.apache.cassandra.db.marshal.*; - -/** - * Extends a column filter (IFilter) to include a number of IndexExpression. - */ -public abstract class ExtendedFilter -{ - private static final Logger logger = LoggerFactory.getLogger(ExtendedFilter.class); - - public final ColumnFamilyStore cfs; - public final long timestamp; - public final DataRange dataRange; - private final int maxResults; - private final boolean countCQL3Rows; - private volatile int currentLimit; - - public static ExtendedFilter create(ColumnFamilyStore cfs, - DataRange dataRange, - List clause, - int maxResults, - boolean countCQL3Rows, - long timestamp) - { - if (clause == null || clause.isEmpty()) - return new EmptyClauseFilter(cfs, dataRange, maxResults, countCQL3Rows, timestamp); - - return new WithClauses(cfs, dataRange, clause, maxResults, countCQL3Rows, timestamp); - } - - protected ExtendedFilter(ColumnFamilyStore cfs, DataRange dataRange, int maxResults, boolean countCQL3Rows, long timestamp) - { - assert cfs != null; - assert dataRange != null; - this.cfs = cfs; - this.dataRange = dataRange; - this.maxResults = maxResults; - this.timestamp = timestamp; - this.countCQL3Rows = countCQL3Rows; - this.currentLimit = maxResults; - if (countCQL3Rows) - dataRange.updateColumnsLimit(maxResults); - } - - public int maxRows() - { - return countCQL3Rows ? Integer.MAX_VALUE : maxResults; - } - - public int maxColumns() - { - return countCQL3Rows ? maxResults : Integer.MAX_VALUE; - } - - public int currentLimit() - { - return currentLimit; - } - - public IDiskAtomFilter columnFilter(ByteBuffer key) - { - return dataRange.columnFilter(key); - } - - public int lastCounted(ColumnFamily data) - { - return dataRange.getLiveCount(data, timestamp); - } - - public void updateFilter(int currentColumnsCount) - { - if (!countCQL3Rows) - return; - - currentLimit = maxResults - currentColumnsCount; - // We propagate that limit to the underlying filter so each internal query don't - // fetch more than we needs it to. - dataRange.updateColumnsLimit(currentLimit); - } - - public abstract List getClause(); - - /** - * Returns a filter to query the columns from the clause that the initial slice filter may not have caught. - * @param data the data retrieve by the initial filter - * @return a filter or null if there can't be any columns we missed with our initial filter (typically if it was a names query, or a slice of the entire row) - */ - public abstract IDiskAtomFilter getExtraFilter(DecoratedKey key, ColumnFamily data); - - /** - * @return data pruned down to the columns originally asked for - */ - public abstract ColumnFamily prune(DecoratedKey key, ColumnFamily data); - - /** Returns true if tombstoned partitions should not be included in results or count towards the limit, false otherwise. */ - public boolean ignoreTombstonedPartitions() - { - return dataRange.ignoredTombstonedPartitions(); - } - - /** - * @return true if the provided data satisfies all the expressions from - * the clause of this filter. - */ - public abstract boolean isSatisfiedBy(DecoratedKey rowKey, ColumnFamily data, Composite prefix, ByteBuffer collectionElement); - - public static boolean satisfies(int comparison, Operator op) - { - switch (op) - { - case EQ: - return comparison == 0; - case GTE: - return comparison >= 0; - case GT: - return comparison > 0; - case LTE: - return comparison <= 0; - case LT: - return comparison < 0; - default: - throw new IllegalStateException(); - } - } - - @Override - public String toString() - { - return Objects.toStringHelper(this) - .add("dataRange", dataRange) - .add("maxResults", maxResults) - .add("currentLimit", currentLimit) - .add("timestamp", timestamp) - .add("countCQL3Rows", countCQL3Rows) - .toString(); - } - - public static class WithClauses extends ExtendedFilter - { - private final List clause; - private final IDiskAtomFilter optimizedFilter; - - public WithClauses(ColumnFamilyStore cfs, - DataRange range, - List clause, - int maxResults, - boolean countCQL3Rows, - long timestamp) - { - super(cfs, range, maxResults, countCQL3Rows, timestamp); - assert clause != null; - this.clause = clause; - this.optimizedFilter = computeOptimizedFilter(); - } - - /* - * Potentially optimize the column filter if we have a change to make it catch all clauses - * right away. - */ - private IDiskAtomFilter computeOptimizedFilter() - { - /* - * We shouldn't do the "optimization" for composites as the index names are not valid column names - * (which the rest of the method assumes). Said optimization is not useful for composites anyway. - * We also don't want to do for paging ranges as the actual filter depends on the row key (it would - * probably be possible to make it work but we won't really use it so we don't bother). - */ - if (cfs.getComparator().isCompound() || dataRange instanceof DataRange.Paging) - return null; - - IDiskAtomFilter filter = dataRange.columnFilter(null); // ok since not a paging range - if (filter instanceof SliceQueryFilter) - { - // if we have a high chance of getting all the columns in a single index slice (and it's not too costly), do that. - // otherwise, the extraFilter (lazily created) will fetch by name the columns referenced by the additional expressions. - if (cfs.metric.maxRowSize.getValue() < DatabaseDescriptor.getColumnIndexSize()) - { - logger.trace("Expanding slice filter to entire row to cover additional expressions"); - return new SliceQueryFilter(ColumnSlice.ALL_COLUMNS_ARRAY, ((SliceQueryFilter)filter).reversed, Integer.MAX_VALUE); - } - } - else - { - logger.trace("adding columns to original Filter to cover additional expressions"); - assert filter instanceof NamesQueryFilter; - if (!clause.isEmpty()) - { - SortedSet columns = new TreeSet(cfs.getComparator()); - for (IndexExpression expr : clause) - columns.add(cfs.getComparator().cellFromByteBuffer(expr.column)); - columns.addAll(((NamesQueryFilter) filter).columns); - return ((NamesQueryFilter) filter).withUpdatedColumns(columns); - } - } - return null; - } - - @Override - public IDiskAtomFilter columnFilter(ByteBuffer key) - { - return optimizedFilter == null ? dataRange.columnFilter(key) : optimizedFilter; - } - - public List getClause() - { - return clause; - } - - /* - * We may need an extra query only if the original query wasn't selecting the row entirely. - * Furthermore, we only need the extra query if we haven't yet got all the expressions from the clause. - */ - private boolean needsExtraQuery(ByteBuffer rowKey, ColumnFamily data) - { - IDiskAtomFilter filter = columnFilter(rowKey); - if (filter instanceof SliceQueryFilter && DataRange.isFullRowSlice((SliceQueryFilter)filter)) - return false; - - for (IndexExpression expr : clause) - { - if (data.getColumn(data.getComparator().cellFromByteBuffer(expr.column)) == null) - { - logger.debug("adding extraFilter to cover additional expressions"); - return true; - } - } - return false; - } - - public IDiskAtomFilter getExtraFilter(DecoratedKey rowKey, ColumnFamily data) - { - /* - * This method assumes the IndexExpression names are valid column names, which is not the - * case with composites. This is ok for now however since: - * 1) CompositeSearcher doesn't use it. - * 2) We don't yet allow non-indexed range slice with filters in CQL3 (i.e. this will never be - * called by CFS.filter() for composites). - */ - assert !(cfs.getComparator().isCompound()) : "Sequential scan with filters is not supported (if you just created an index, you " - + "need to wait for the creation to be propagated to all nodes before querying it)"; - - if (!needsExtraQuery(rowKey.getKey(), data)) - return null; - - // Note: for counters we must be careful to not add a column that was already there (to avoid overcount). That is - // why we do the dance of avoiding to query any column we already have (it's also more efficient anyway) - SortedSet columns = new TreeSet(cfs.getComparator()); - for (IndexExpression expr : clause) - { - CellName name = data.getComparator().cellFromByteBuffer(expr.column); - if (data.getColumn(name) == null) - columns.add(name); - } - assert !columns.isEmpty(); - return new NamesQueryFilter(columns); - } - - public ColumnFamily prune(DecoratedKey rowKey, ColumnFamily data) - { - if (optimizedFilter == null) - return data; - - ColumnFamily pruned = data.cloneMeShallow(); - IDiskAtomFilter filter = dataRange.columnFilter(rowKey.getKey()); - Iterator iter = filter.getColumnIterator(data); - try - { - filter.collectReducedColumns(pruned, QueryFilter.gatherTombstones(pruned, iter), rowKey, cfs.gcBefore(timestamp), timestamp); - } - catch (TombstoneOverwhelmingException e) - { - e.setKey(rowKey); - throw e; - } - return pruned; - } - - public boolean isSatisfiedBy(DecoratedKey rowKey, ColumnFamily data, Composite prefix, ByteBuffer collectionElement) - { - for (IndexExpression expression : clause) - { - ColumnDefinition def = data.metadata().getColumnDefinition(expression.column); - ByteBuffer dataValue = null; - AbstractType validator = null; - if (def == null) - { - // This can't happen with CQL3 as this should be rejected upfront. For thrift however, - // cell name are not predefined. But that means the cell name correspond to an internal one. - Cell cell = data.getColumn(data.getComparator().cellFromByteBuffer(expression.column)); - if (cell != null) - { - dataValue = cell.value(); - validator = data.metadata().getDefaultValidator(); - } - } - else - { - if (def.type.isCollection() && def.type.isMultiCell()) - { - if (!collectionSatisfies(def, data, prefix, expression)) - return false; - continue; - } - - dataValue = extractDataValue(def, rowKey.getKey(), data, prefix); - validator = def.type; - } - - if (dataValue == null) - return false; - - if (expression.operator == Operator.CONTAINS) - { - assert def != null && def.type.isCollection() && !def.type.isMultiCell(); - CollectionType type = (CollectionType)def.type; - switch (type.kind) - { - case LIST: - ListType listType = (ListType)def.type; - if (!listType.getSerializer().deserialize(dataValue).contains(listType.getElementsType().getSerializer().deserialize(expression.value))) - return false; - break; - case SET: - SetType setType = (SetType)def.type; - if (!setType.getSerializer().deserialize(dataValue).contains(setType.getElementsType().getSerializer().deserialize(expression.value))) - return false; - break; - case MAP: - MapType mapType = (MapType)def.type; - if (!mapType.getSerializer().deserialize(dataValue).containsValue(mapType.getValuesType().getSerializer().deserialize(expression.value))) - return false; - break; - } - } - else if (expression.operator == Operator.CONTAINS_KEY) - { - assert def != null && def.type.isCollection() && !def.type.isMultiCell() && def.type instanceof MapType; - MapType mapType = (MapType)def.type; - if (mapType.getSerializer().getSerializedValue(dataValue, expression.value, mapType.getKeysType()) == null) - return false; - } - else - { - int v = validator.compare(dataValue, expression.value); - if (!satisfies(v, expression.operator)) - return false; - } - } - return true; - } - - private static boolean collectionSatisfies(ColumnDefinition def, ColumnFamily data, Composite prefix, IndexExpression expr) - { - assert def.type.isCollection() && def.type.isMultiCell(); - CollectionType type = (CollectionType)def.type; - - if (expr.isContains()) - { - // get a slice of the collection cells - Iterator iter = data.iterator(new ColumnSlice[]{ data.getComparator().create(prefix, def).slice() }); - while (iter.hasNext()) - { - Cell cell = iter.next(); - if (type.kind == CollectionType.Kind.SET) - { - if (type.nameComparator().compare(cell.name().collectionElement(), expr.value) == 0) - return true; - } - else - { - if (type.valueComparator().compare(cell.value(), expr.value) == 0) - return true; - } - } - - return false; - } - - assert type.kind == CollectionType.Kind.MAP; - if (expr.isContainsKey()) - return data.getColumn(data.getComparator().create(prefix, def, expr.value)) != null; - - Iterator iter = data.iterator(new ColumnSlice[]{ data.getComparator().create(prefix, def).slice() }); - ByteBuffer key = CompositeType.extractComponent(expr.value, 0); - ByteBuffer value = CompositeType.extractComponent(expr.value, 1); - while (iter.hasNext()) - { - Cell next = iter.next(); - if (type.nameComparator().compare(next.name().collectionElement(), key) == 0 && - type.valueComparator().compare(next.value(), value) == 0) - return true; - } - return false; - } - - private ByteBuffer extractDataValue(ColumnDefinition def, ByteBuffer rowKey, ColumnFamily data, Composite prefix) - { - switch (def.kind) - { - case PARTITION_KEY: - return def.isOnAllComponents() - ? rowKey - : ((CompositeType)data.metadata().getKeyValidator()).split(rowKey)[def.position()]; - case CLUSTERING_COLUMN: - return prefix.get(def.position()); - case REGULAR: - CellName cname = prefix == null - ? data.getComparator().cellFromByteBuffer(def.name.bytes) - : data.getComparator().create(prefix, def); - - Cell cell = data.getColumn(cname); - return cell == null ? null : cell.value(); - case COMPACT_VALUE: - assert data.getColumnCount() == 1; - return data.getSortedColumns().iterator().next().value(); - } - throw new AssertionError(); - } - - @Override - public String toString() - { - return Objects.toStringHelper(this) - .add("dataRange", dataRange) - .add("timestamp", timestamp) - .add("clause", clause) - .toString(); - } - } - - private static class EmptyClauseFilter extends ExtendedFilter - { - public EmptyClauseFilter(ColumnFamilyStore cfs, DataRange range, int maxResults, boolean countCQL3Rows, long timestamp) - { - super(cfs, range, maxResults, countCQL3Rows, timestamp); - } - - public List getClause() - { - return Collections.emptyList(); - } - - public IDiskAtomFilter getExtraFilter(DecoratedKey key, ColumnFamily data) - { - return null; - } - - public ColumnFamily prune(DecoratedKey rowKey, ColumnFamily data) - { - return data; - } - - public boolean isSatisfiedBy(DecoratedKey rowKey, ColumnFamily data, Composite prefix, ByteBuffer collectionElement) - { - return true; - } - } -} diff --git a/src/java/org/apache/cassandra/db/filter/IDiskAtomFilter.java b/src/java/org/apache/cassandra/db/filter/IDiskAtomFilter.java deleted file mode 100644 index a541d5e7ee..0000000000 --- a/src/java/org/apache/cassandra/db/filter/IDiskAtomFilter.java +++ /dev/null @@ -1,146 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.apache.cassandra.db.filter; - -import java.io.DataInput; -import java.io.IOException; -import java.util.Comparator; -import java.util.Iterator; - -import org.apache.cassandra.db.*; -import org.apache.cassandra.db.columniterator.OnDiskAtomIterator; -import org.apache.cassandra.db.composites.CellNameType; -import org.apache.cassandra.db.composites.Composite; -import org.apache.cassandra.db.composites.CType; -import org.apache.cassandra.io.IVersionedSerializer; -import org.apache.cassandra.io.sstable.format.SSTableReader; -import org.apache.cassandra.io.util.DataOutputPlus; -import org.apache.cassandra.io.util.FileDataInput; - -/** - * Given an implementation-specific description of what columns to look for, provides methods - * to extract the desired columns from a Memtable, SSTable, or SuperColumn. Either the get*ColumnIterator - * methods will be called, or filterSuperColumn, but not both on the same object. QueryFilter - * takes care of putting the two together if subcolumn filtering needs to be done, based on the - * querypath that it knows (but that IFilter implementations are oblivious to). - */ -public interface IDiskAtomFilter -{ - /** - * returns an iterator that returns columns from the given columnFamily - * matching the Filter criteria in sorted order. - */ - public Iterator getColumnIterator(ColumnFamily cf); - - public OnDiskAtomIterator getColumnIterator(DecoratedKey key, ColumnFamily cf); - - /** - * Get an iterator that returns columns from the given SSTable using the opened file - * matching the Filter criteria in sorted order. - * @param sstable - * @param file Already opened file data input, saves us opening another one - * @param key The key of the row we are about to iterate over - */ - public OnDiskAtomIterator getSSTableColumnIterator(SSTableReader sstable, FileDataInput file, DecoratedKey key, RowIndexEntry indexEntry); - - /** - * returns an iterator that returns columns from the given SSTable - * matching the Filter criteria in sorted order. - */ - public OnDiskAtomIterator getSSTableColumnIterator(SSTableReader sstable, DecoratedKey key); - - /** - * collects columns from reducedColumns into returnCF. Termination is determined - * by the filter code, which should have some limit on the number of columns - * to avoid running out of memory on large rows. - */ - public void collectReducedColumns(ColumnFamily container, Iterator reducedColumns, DecoratedKey key, int gcBefore, long now); - - public Comparator getColumnComparator(CellNameType comparator); - - public boolean isReversed(); - public void updateColumnsLimit(int newLimit); - - public int getLiveCount(ColumnFamily cf, long now); - public ColumnCounter columnCounter(CellNameType comparator, long now); - - public IDiskAtomFilter cloneShallow(); - public boolean maySelectPrefix(CType type, Composite prefix); - - public boolean shouldInclude(SSTableReader sstable); - - public boolean countCQL3Rows(CellNameType comparator); - - public boolean isHeadFilter(); - - /** - * Whether the provided cf, that is assumed to contain the head of the - * partition, contains enough data to cover this filter. - */ - public boolean isFullyCoveredBy(ColumnFamily cf, long now); - - public static class Serializer implements IVersionedSerializer - { - private final CellNameType type; - - public Serializer(CellNameType type) - { - this.type = type; - } - - public void serialize(IDiskAtomFilter filter, DataOutputPlus out, int version) throws IOException - { - if (filter instanceof SliceQueryFilter) - { - out.writeByte(0); - type.sliceQueryFilterSerializer().serialize((SliceQueryFilter)filter, out, version); - } - else - { - out.writeByte(1); - type.namesQueryFilterSerializer().serialize((NamesQueryFilter)filter, out, version); - } - } - - public IDiskAtomFilter deserialize(DataInput in, int version) throws IOException - { - int b = in.readByte(); - if (b == 0) - { - return type.sliceQueryFilterSerializer().deserialize(in, version); - } - else - { - assert b == 1; - return type.namesQueryFilterSerializer().deserialize(in, version); - } - } - - public long serializedSize(IDiskAtomFilter filter, int version) - { - int size = 1; - if (filter instanceof SliceQueryFilter) - size += type.sliceQueryFilterSerializer().serializedSize((SliceQueryFilter)filter, version); - else - size += type.namesQueryFilterSerializer().serializedSize((NamesQueryFilter)filter, version); - return size; - } - } - - public Iterator getRangeTombstoneIterator(ColumnFamily source); -} diff --git a/src/java/org/apache/cassandra/db/filter/NamesQueryFilter.java b/src/java/org/apache/cassandra/db/filter/NamesQueryFilter.java deleted file mode 100644 index c8f63bb258..0000000000 --- a/src/java/org/apache/cassandra/db/filter/NamesQueryFilter.java +++ /dev/null @@ -1,301 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.apache.cassandra.db.filter; - -import java.io.DataInput; -import java.io.IOException; -import java.util.Comparator; -import java.util.Iterator; -import java.util.SortedSet; -import java.util.TreeSet; - -import org.apache.cassandra.io.sstable.format.SSTableReader; -import org.apache.commons.lang3.StringUtils; -import com.google.common.collect.AbstractIterator; -import com.google.common.collect.Iterators; - -import org.apache.cassandra.db.*; -import org.apache.cassandra.db.columniterator.OnDiskAtomIterator; -import org.apache.cassandra.db.composites.CellName; -import org.apache.cassandra.db.composites.CellNameType; -import org.apache.cassandra.db.composites.Composite; -import org.apache.cassandra.db.composites.CType; -import org.apache.cassandra.io.ISerializer; -import org.apache.cassandra.io.IVersionedSerializer; -import org.apache.cassandra.io.util.DataOutputPlus; -import org.apache.cassandra.io.util.FileDataInput; -import org.apache.cassandra.utils.SearchIterator; - -public class NamesQueryFilter implements IDiskAtomFilter -{ - public final SortedSet columns; - - // If true, getLiveCount will always return either 0 or 1. This uses the fact that we know - // CQL3 will never use a name filter with cell names spanning multiple CQL3 rows. - private final boolean countCQL3Rows; - - public NamesQueryFilter(SortedSet columns) - { - this(columns, false); - } - - public NamesQueryFilter(SortedSet columns, boolean countCQL3Rows) - { - this.columns = columns; - this.countCQL3Rows = countCQL3Rows; - } - - public NamesQueryFilter cloneShallow() - { - // NQF is immutable as far as shallow cloning is concerned, so save the allocation. - return this; - } - - public NamesQueryFilter withUpdatedColumns(SortedSet newColumns) - { - return new NamesQueryFilter(newColumns, countCQL3Rows); - } - - @SuppressWarnings("unchecked") - public Iterator getColumnIterator(ColumnFamily cf) - { - assert cf != null; - return (Iterator) (Iterator) new ByNameColumnIterator(columns.iterator(), null, cf); - } - - public OnDiskAtomIterator getColumnIterator(DecoratedKey key, ColumnFamily cf) - { - assert cf != null; - return new ByNameColumnIterator(columns.iterator(), key, cf); - } - - public OnDiskAtomIterator getSSTableColumnIterator(SSTableReader sstable, DecoratedKey key) - { - return sstable.iterator(key, columns); - } - - public OnDiskAtomIterator getSSTableColumnIterator(SSTableReader sstable, FileDataInput file, DecoratedKey key, RowIndexEntry indexEntry) - { - return sstable.iterator(file, key, columns, indexEntry); - } - - public void collectReducedColumns(ColumnFamily container, Iterator reducedColumns, DecoratedKey key, int gcBefore, long now) - { - DeletionInfo.InOrderTester tester = container.inOrderDeletionTester(); - while (reducedColumns.hasNext()) - container.maybeAppendColumn(reducedColumns.next(), tester, gcBefore); - } - - public Comparator getColumnComparator(CellNameType comparator) - { - return comparator.columnComparator(false); - } - - @Override - public String toString() - { - return "NamesQueryFilter(" + - "columns=" + StringUtils.join(columns, ",") + - ')'; - } - - public boolean isReversed() - { - return false; - } - - public void updateColumnsLimit(int newLimit) - { - } - - public int getLiveCount(ColumnFamily cf, long now) - { - // Note: we could use columnCounter() but we save the object allocation as it's simple enough - - if (countCQL3Rows) - return cf.hasOnlyTombstones(now) ? 0 : 1; - - int count = 0; - for (Cell cell : cf) - { - if (cell.isLive(now)) - count++; - } - return count; - } - - public boolean maySelectPrefix(CType type, Composite prefix) - { - for (CellName column : columns) - { - if (prefix.isPrefixOf(type, column)) - return true; - } - return false; - } - - public boolean shouldInclude(SSTableReader sstable) - { - return true; - } - - public boolean isFullyCoveredBy(ColumnFamily cf, long now) - { - // cf will cover all the requested columns if the range it covers include - // all said columns - CellName first = cf.iterator(ColumnSlice.ALL_COLUMNS_ARRAY).next().name(); - CellName last = cf.reverseIterator(ColumnSlice.ALL_COLUMNS_ARRAY).next().name(); - - return cf.getComparator().compare(first, columns.first()) <= 0 - && cf.getComparator().compare(columns.last(), last) <= 0; - } - - public boolean isHeadFilter() - { - return false; - } - - public boolean countCQL3Rows(CellNameType comparator) - { - return countCQL3Rows; - } - - public boolean countCQL3Rows() - { - return countCQL3Rows(null); - } - - public ColumnCounter columnCounter(CellNameType comparator, long now) - { - return countCQL3Rows - ? new ColumnCounter.GroupByPrefix(now, null, 0) - : new ColumnCounter(now); - } - - private static class ByNameColumnIterator extends AbstractIterator implements OnDiskAtomIterator - { - private final ColumnFamily cf; - private final DecoratedKey key; - private final Iterator names; - private final SearchIterator cells; - - public ByNameColumnIterator(Iterator names, DecoratedKey key, ColumnFamily cf) - { - this.names = names; - this.cf = cf; - this.key = key; - this.cells = cf.searchIterator(); - } - - protected OnDiskAtom computeNext() - { - while (names.hasNext() && cells.hasNext()) - { - CellName current = names.next(); - Cell cell = cells.next(current); - if (cell != null) - return cell; - } - return endOfData(); - } - - public ColumnFamily getColumnFamily() - { - return cf; - } - - public DecoratedKey getKey() - { - return key; - } - - public void close() throws IOException { } - } - - public static class Serializer implements IVersionedSerializer - { - private CellNameType type; - - public Serializer(CellNameType type) - { - this.type = type; - } - - public void serialize(NamesQueryFilter f, DataOutputPlus out, int version) throws IOException - { - out.writeInt(f.columns.size()); - ISerializer serializer = type.cellSerializer(); - for (CellName cName : f.columns) - { - serializer.serialize(cName, out); - } - out.writeBoolean(f.countCQL3Rows); - } - - public NamesQueryFilter deserialize(DataInput in, int version) throws IOException - { - int size = in.readInt(); - SortedSet columns = new TreeSet<>(type); - ISerializer serializer = type.cellSerializer(); - for (int i = 0; i < size; ++i) - columns.add(serializer.deserialize(in)); - boolean countCQL3Rows = in.readBoolean(); - return new NamesQueryFilter(columns, countCQL3Rows); - } - - public long serializedSize(NamesQueryFilter f, int version) - { - TypeSizes sizes = TypeSizes.NATIVE; - int size = sizes.sizeof(f.columns.size()); - ISerializer serializer = type.cellSerializer(); - for (CellName cName : f.columns) - size += serializer.serializedSize(cName, sizes); - size += sizes.sizeof(f.countCQL3Rows); - return size; - } - } - - public Iterator getRangeTombstoneIterator(final ColumnFamily source) - { - if (!source.deletionInfo().hasRanges()) - return Iterators.emptyIterator(); - - return new AbstractIterator() - { - private final Iterator names = columns.iterator(); - private RangeTombstone lastFindRange; - - protected RangeTombstone computeNext() - { - while (names.hasNext()) - { - CellName next = names.next(); - if (lastFindRange != null && lastFindRange.includes(source.getComparator(), next)) - return lastFindRange; - - // We keep the last range around as since names are in sort order, it's - // possible it will match the next name too. - lastFindRange = source.deletionInfo().rangeCovering(next); - if (lastFindRange != null) - return lastFindRange; - } - return endOfData(); - } - }; - } -} diff --git a/src/java/org/apache/cassandra/db/filter/QueryFilter.java b/src/java/org/apache/cassandra/db/filter/QueryFilter.java deleted file mode 100644 index 15ee33df2f..0000000000 --- a/src/java/org/apache/cassandra/db/filter/QueryFilter.java +++ /dev/null @@ -1,262 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.apache.cassandra.db.filter; - -import java.util.ArrayList; -import java.util.Comparator; -import java.util.Iterator; -import java.util.List; -import java.util.SortedSet; - -import org.apache.cassandra.db.Cell; -import org.apache.cassandra.db.ColumnFamily; -import org.apache.cassandra.db.DecoratedKey; -import org.apache.cassandra.db.DeletionInfo; -import org.apache.cassandra.db.OnDiskAtom; -import org.apache.cassandra.db.RangeTombstone; -import org.apache.cassandra.db.columniterator.IdentityQueryFilter; -import org.apache.cassandra.db.columniterator.OnDiskAtomIterator; -import org.apache.cassandra.db.composites.CellName; -import org.apache.cassandra.db.composites.Composite; -import org.apache.cassandra.io.sstable.format.SSTableReader; -import org.apache.cassandra.utils.MergeIterator; - -public class QueryFilter -{ - public final DecoratedKey key; - public final String cfName; - public final IDiskAtomFilter filter; - public final long timestamp; - - public QueryFilter(DecoratedKey key, String cfName, IDiskAtomFilter filter, long timestamp) - { - this.key = key; - this.cfName = cfName; - this.filter = filter; - this.timestamp = timestamp; - } - - public Iterator getIterator(ColumnFamily cf) - { - assert cf != null; - return filter.getColumnIterator(cf); - } - - public OnDiskAtomIterator getSSTableColumnIterator(SSTableReader sstable) - { - return filter.getSSTableColumnIterator(sstable, key); - } - - public void collateOnDiskAtom(ColumnFamily returnCF, - List> toCollate, - int gcBefore) - { - collateOnDiskAtom(returnCF, toCollate, filter, this.key, gcBefore, timestamp); - } - - public static void collateOnDiskAtom(ColumnFamily returnCF, - List> toCollate, - IDiskAtomFilter filter, - DecoratedKey key, - int gcBefore, - long timestamp) - { - List> filteredIterators = new ArrayList<>(toCollate.size()); - for (Iterator iter : toCollate) - filteredIterators.add(gatherTombstones(returnCF, iter)); - collateColumns(returnCF, filteredIterators, filter, key, gcBefore, timestamp); - } - - // When there is only a single source of atoms, we can skip the collate step - public void collateOnDiskAtom(ColumnFamily returnCF, Iterator toCollate, int gcBefore) - { - filter.collectReducedColumns(returnCF, gatherTombstones(returnCF, toCollate), this.key, gcBefore, timestamp); - } - - public void collateColumns(ColumnFamily returnCF, List> toCollate, int gcBefore) - { - collateColumns(returnCF, toCollate, filter, this.key, gcBefore, timestamp); - } - - public static void collateColumns(ColumnFamily returnCF, - List> toCollate, - IDiskAtomFilter filter, - DecoratedKey key, - int gcBefore, - long timestamp) - { - Comparator comparator = filter.getColumnComparator(returnCF.getComparator()); - - Iterator reduced = toCollate.size() == 1 - ? toCollate.get(0) - : MergeIterator.get(toCollate, comparator, getReducer(comparator)); - - filter.collectReducedColumns(returnCF, reduced, key, gcBefore, timestamp); - } - - private static MergeIterator.Reducer getReducer(final Comparator comparator) - { - // define a 'reduced' iterator that merges columns w/ the same name, which - // greatly simplifies computing liveColumns in the presence of tombstones. - return new MergeIterator.Reducer() - { - Cell current; - - public void reduce(Cell next) - { - assert current == null || comparator.compare(current, next) == 0; - current = current == null ? next : current.reconcile(next); - } - - protected Cell getReduced() - { - assert current != null; - Cell toReturn = current; - current = null; - return toReturn; - } - - @Override - public boolean trivialReduceIsTrivial() - { - return true; - } - }; - } - - /** - * Given an iterator of on disk atom, returns an iterator that filters the tombstone range - * markers adding them to {@code returnCF} and returns the normal column. - */ - public static Iterator gatherTombstones(final ColumnFamily returnCF, final Iterator iter) - { - return new Iterator() - { - private Cell next; - - public boolean hasNext() - { - if (next != null) - return true; - - getNext(); - return next != null; - } - - public Cell next() - { - if (next == null) - getNext(); - - assert next != null; - Cell toReturn = next; - next = null; - return toReturn; - } - - private void getNext() - { - while (iter.hasNext()) - { - OnDiskAtom atom = iter.next(); - - if (atom instanceof Cell) - { - next = (Cell)atom; - break; - } - else - { - returnCF.addAtom(atom); - } - } - } - - public void remove() - { - throw new UnsupportedOperationException(); - } - }; - } - - public String getColumnFamilyName() - { - return cfName; - } - - /** - * @return a QueryFilter object to satisfy the given slice criteria: - * @param key the row to slice - * @param cfName column family to query - * @param start column to start slice at, inclusive; empty for "the first column" - * @param finish column to stop slice at, inclusive; empty for "the last column" - * @param reversed true to start with the largest column (as determined by configured sort order) instead of smallest - * @param limit maximum number of non-deleted columns to return - * @param timestamp time to use for determining expiring columns' state - */ - public static QueryFilter getSliceFilter(DecoratedKey key, - String cfName, - Composite start, - Composite finish, - boolean reversed, - int limit, - long timestamp) - { - return new QueryFilter(key, cfName, new SliceQueryFilter(start, finish, reversed, limit), timestamp); - } - - /** - * return a QueryFilter object that includes every column in the row. - * This is dangerous on large rows; avoid except for test code. - */ - public static QueryFilter getIdentityFilter(DecoratedKey key, String cfName, long timestamp) - { - return new QueryFilter(key, cfName, new IdentityQueryFilter(), timestamp); - } - - /** - * @return a QueryFilter object that will return columns matching the given names - * @param key the row to slice - * @param cfName column family to query - * @param columns the column names to restrict the results to, sorted in comparator order - */ - public static QueryFilter getNamesFilter(DecoratedKey key, String cfName, SortedSet columns, long timestamp) - { - return new QueryFilter(key, cfName, new NamesQueryFilter(columns), timestamp); - } - - @Override - public String toString() - { - return getClass().getSimpleName() + "(key=" + key + ", cfName=" + cfName + (filter == null ? "" : ", filter=" + filter) + ")"; - } - - public boolean shouldInclude(SSTableReader sstable) - { - return filter.shouldInclude(sstable); - } - - public void delete(DeletionInfo target, ColumnFamily source) - { - target.add(source.deletionInfo().getTopLevelDeletion()); - // source is the CF currently in the memtable, and it can be large compared to what the filter selects, - // so only consider those range tombstones that the filter do select. - for (Iterator iter = filter.getRangeTombstoneIterator(source); iter.hasNext(); ) - target.add(iter.next(), source.getComparator()); - } -} diff --git a/src/java/org/apache/cassandra/db/filter/RowFilter.java b/src/java/org/apache/cassandra/db/filter/RowFilter.java new file mode 100644 index 0000000000..aff8d16227 --- /dev/null +++ b/src/java/org/apache/cassandra/db/filter/RowFilter.java @@ -0,0 +1,784 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.cassandra.db.filter; + +import java.io.DataInput; +import java.io.IOException; +import java.nio.ByteBuffer; +import java.util.*; + +import com.google.common.base.Objects; + +import org.apache.cassandra.config.CFMetaData; +import org.apache.cassandra.config.ColumnDefinition; +import org.apache.cassandra.cql3.Operator; +import org.apache.cassandra.db.*; +import org.apache.cassandra.db.rows.*; +import org.apache.cassandra.db.partitions.*; +import org.apache.cassandra.db.marshal.*; +import org.apache.cassandra.db.partitions.UnfilteredPartitionIterator; +import org.apache.cassandra.exceptions.InvalidRequestException; +import org.apache.cassandra.io.util.DataOutputPlus; +import org.apache.cassandra.net.MessagingService; +import org.apache.cassandra.utils.ByteBufferUtil; +import org.apache.cassandra.utils.FBUtilities; + +import static org.apache.cassandra.cql3.statements.RequestValidations.*; + +/** + * A filter on which rows a given query should include or exclude. + *

+ * This corresponds to the restrictions on rows that are not handled by the query + * {@link ClusteringIndexFilter}. Some of the expressions of this filter may + * be handled by a 2ndary index, and the rest is simply filtered out from the + * result set (the later can only happen if the query was using ALLOW FILTERING). + */ +public abstract class RowFilter implements Iterable +{ + public static final Serializer serializer = new Serializer(); + public static final RowFilter NONE = new CQLFilter(Collections.emptyList()); + + protected final List expressions; + + protected RowFilter(List expressions) + { + this.expressions = expressions; + } + + public static RowFilter create() + { + return new CQLFilter(new ArrayList()); + } + + public static RowFilter create(int capacity) + { + return new CQLFilter(new ArrayList(capacity)); + } + + public static RowFilter forThrift(int capacity) + { + return new ThriftFilter(new ArrayList(capacity)); + } + + public void add(ColumnDefinition def, Operator op, ByteBuffer value) + { + expressions.add(new SimpleExpression(def, op, value)); + } + + public void addMapEquality(ColumnDefinition def, ByteBuffer key, Operator op, ByteBuffer value) + { + expressions.add(new MapEqualityExpression(def, key, op, value)); + } + + public void addThriftExpression(CFMetaData metadata, ByteBuffer name, Operator op, ByteBuffer value) + { + assert (this instanceof ThriftFilter); + expressions.add(new ThriftExpression(metadata, name, op, value)); + } + + /** + * Filters the provided iterator so that only the row satisfying the expression of this filter + * are included in the resulting iterator. + * + * @param iter the iterator to filter + * @param nowInSec the time of query in seconds. + * @return the filtered iterator. + */ + public abstract UnfilteredPartitionIterator filter(UnfilteredPartitionIterator iter, int nowInSec); + + /** + * Returns this filter but without the provided expression. This method + * *assumes* that the filter contains the provided expression. + */ + public RowFilter without(Expression expression) + { + assert expressions.contains(expression); + if (expressions.size() == 1) + return RowFilter.NONE; + + List newExpressions = new ArrayList<>(expressions.size() - 1); + for (Expression e : expressions) + if (!e.equals(expression)) + newExpressions.add(e); + + return withNewExpressions(newExpressions); + } + + protected abstract RowFilter withNewExpressions(List expressions); + + public boolean isEmpty() + { + return expressions.isEmpty(); + } + + public Iterator iterator() + { + return expressions.iterator(); + } + + private static Clustering makeCompactClustering(CFMetaData metadata, ByteBuffer name) + { + assert metadata.isCompactTable(); + if (metadata.isCompound()) + { + List values = CompositeType.splitName(name); + return new SimpleClustering(values.toArray(new ByteBuffer[metadata.comparator.size()])); + } + else + { + return new SimpleClustering(name); + } + } + + @Override + public String toString() + { + StringBuilder sb = new StringBuilder(); + for (int i = 0; i < expressions.size(); i++) + { + if (i > 0) + sb.append(" AND "); + sb.append(expressions.get(i)); + } + return sb.toString(); + } + + private static class CQLFilter extends RowFilter + { + private CQLFilter(List expressions) + { + super(expressions); + } + + public UnfilteredPartitionIterator filter(UnfilteredPartitionIterator iter, final int nowInSec) + { + if (expressions.isEmpty()) + return iter; + + return new WrappingUnfilteredPartitionIterator(iter) + { + @Override + public UnfilteredRowIterator computeNext(final UnfilteredRowIterator iter) + { + return new FilteringRowIterator(iter) + { + // We filter tombstones when passing the row to isSatisfiedBy so that the method doesn't have to bother with them. + // (we should however not filter them in the output of the method, hence it's not used as row filter for the + // FilteringRowIterator) + private final TombstoneFilteringRow filter = new TombstoneFilteringRow(nowInSec); + + protected boolean includeRow(Row row) + { + return CQLFilter.this.isSatisfiedBy(iter.partitionKey(), filter.setTo(row)); + } + }; + } + }; + } + + /** + * Returns whether the provided row (with it's partition key) satisfies + * this row filter or not (that is, if it satisfies all of its expressions). + */ + private boolean isSatisfiedBy(DecoratedKey partitionKey, Row row) + { + for (Expression e : expressions) + if (!e.isSatisfiedBy(partitionKey, row)) + return false; + + return true; + } + + protected RowFilter withNewExpressions(List expressions) + { + return new CQLFilter(expressions); + } + } + + private static class ThriftFilter extends RowFilter + { + private ThriftFilter(List expressions) + { + super(expressions); + } + + public UnfilteredPartitionIterator filter(UnfilteredPartitionIterator iter, final int nowInSec) + { + if (expressions.isEmpty()) + return iter; + + return new WrappingUnfilteredPartitionIterator(iter) + { + @Override + public UnfilteredRowIterator computeNext(final UnfilteredRowIterator iter) + { + // Thrift does not filter rows, it filters entire partition if any of the expression is not + // satisfied, which forces us to materialize the result (in theory we could materialize only + // what we need which might or might not be everything, but we keep it simple since in practice + // it's not worth that it has ever been). + ArrayBackedPartition result = ArrayBackedPartition.create(iter); + + // The partition needs to have a row for every expression, and the expression needs to be valid. + for (Expression expr : expressions) + { + assert expr instanceof ThriftExpression; + Row row = result.getRow(makeCompactClustering(iter.metadata(), expr.column().name.bytes)); + if (row == null || !expr.isSatisfiedBy(iter.partitionKey(), row)) + return null; + } + // If we get there, it means all expressions where satisfied, so return the original result + return result.unfilteredIterator(); + } + }; + } + + protected RowFilter withNewExpressions(List expressions) + { + return new ThriftFilter(expressions); + } + } + + public static abstract class Expression + { + private static final Serializer serializer = new Serializer(); + + // Note: the order of this enum matter, it's used for serialization + protected enum Kind { SIMPLE, MAP_EQUALITY, THRIFT_DYN_EXPR } + + abstract Kind kind(); + protected final ColumnDefinition column; + protected final Operator operator; + protected final ByteBuffer value; + + protected Expression(ColumnDefinition column, Operator operator, ByteBuffer value) + { + this.column = column; + this.operator = operator; + this.value = value; + } + + public ColumnDefinition column() + { + return column; + } + + public Operator operator() + { + return operator; + } + + /** + * Checks if the operator of this IndexExpression is a CONTAINS operator. + * + * @return true if the operator of this IndexExpression is a CONTAINS + * operator, false otherwise. + */ + public boolean isContains() + { + return Operator.CONTAINS == operator; + } + + /** + * Checks if the operator of this IndexExpression is a CONTAINS_KEY operator. + * + * @return true if the operator of this IndexExpression is a CONTAINS_KEY + * operator, false otherwise. + */ + public boolean isContainsKey() + { + return Operator.CONTAINS_KEY == operator; + } + + /** + * If this expression is used to query an index, the value to use as + * partition key for that index query. + */ + public ByteBuffer getIndexValue() + { + return value; + } + + public void validateForIndexing() throws InvalidRequestException + { + checkNotNull(value, "Unsupported null value for indexed column %s", column.name); + checkBindValueSet(value, "Unsupported unset value for indexed column %s", column.name); + checkFalse(value.remaining() > FBUtilities.MAX_UNSIGNED_SHORT, "Index expression values may not be larger than 64K"); + } + + /** + * Returns whether the provided row satisfied this expression or not. + * + * @param partitionKey the partition key for row to check. + * @param row the row to check. It should *not* contain deleted cells + * (i.e. it should come from a RowIterator). + * @return whether the row is satisfied by this expression. + */ + public abstract boolean isSatisfiedBy(DecoratedKey partitionKey, Row row); + + protected ByteBuffer getValue(DecoratedKey partitionKey, Row row) + { + switch (column.kind) + { + case PARTITION_KEY: + return column.isOnAllComponents() + ? partitionKey.getKey() + : CompositeType.extractComponent(partitionKey.getKey(), column.position()); + case CLUSTERING_COLUMN: + return row.clustering().get(column.position()); + default: + Cell cell = row.getCell(column); + return cell == null ? null : cell.value(); + } + } + + @Override + public boolean equals(Object o) + { + if (this == o) + return true; + + if (!(o instanceof Expression)) + return false; + + Expression that = (Expression)o; + + return Objects.equal(this.kind(), that.kind()) + && Objects.equal(this.column.name, that.column.name) + && Objects.equal(this.operator, that.operator) + && Objects.equal(this.value, that.value); + } + + @Override + public int hashCode() + { + return Objects.hashCode(column.name, operator, value); + } + + private static class Serializer + { + public void serialize(Expression expression, DataOutputPlus out, int version) throws IOException + { + ByteBufferUtil.writeWithShortLength(expression.column.name.bytes, out); + expression.operator.writeTo(out); + + if (version >= MessagingService.VERSION_30) + out.writeByte(expression.kind().ordinal()); + + switch (expression.kind()) + { + case SIMPLE: + ByteBufferUtil.writeWithShortLength(((SimpleExpression)expression).value, out); + break; + case MAP_EQUALITY: + MapEqualityExpression mexpr = (MapEqualityExpression)expression; + if (version < MessagingService.VERSION_30) + { + ByteBufferUtil.writeWithShortLength(mexpr.getIndexValue(), out); + } + else + { + ByteBufferUtil.writeWithShortLength(mexpr.key, out); + ByteBufferUtil.writeWithShortLength(mexpr.value, out); + } + break; + case THRIFT_DYN_EXPR: + ByteBufferUtil.writeWithShortLength(((ThriftExpression)expression).value, out); + break; + } + } + + public Expression deserialize(DataInput in, int version, CFMetaData metadata) throws IOException + { + ByteBuffer name = ByteBufferUtil.readWithShortLength(in); + Operator operator = Operator.readFrom(in); + + ColumnDefinition column = metadata.getColumnDefinition(name); + if (!metadata.isCompactTable() && column == null) + throw new RuntimeException("Unknown (or dropped) column " + UTF8Type.instance.getString(name) + " during deserialization"); + + Kind kind; + if (version >= MessagingService.VERSION_30) + { + kind = Kind.values()[in.readByte()]; + } + else + { + if (column == null) + kind = Kind.THRIFT_DYN_EXPR; + else if (column.type instanceof MapType && operator == Operator.EQ) + kind = Kind.MAP_EQUALITY; + else + kind = Kind.SIMPLE; + } + + switch (kind) + { + case SIMPLE: + return new SimpleExpression(column, operator, ByteBufferUtil.readWithShortLength(in)); + case MAP_EQUALITY: + ByteBuffer key, value; + if (version < MessagingService.VERSION_30) + { + ByteBuffer composite = ByteBufferUtil.readWithShortLength(in); + key = CompositeType.extractComponent(composite, 0); + value = CompositeType.extractComponent(composite, 0); + } + else + { + key = ByteBufferUtil.readWithShortLength(in); + value = ByteBufferUtil.readWithShortLength(in); + } + return new MapEqualityExpression(column, key, operator, value); + case THRIFT_DYN_EXPR: + return new ThriftExpression(metadata, name, operator, ByteBufferUtil.readWithShortLength(in)); + } + throw new AssertionError(); + } + + public long serializedSize(Expression expression, int version) + { + TypeSizes sizes = TypeSizes.NATIVE; + long size = ByteBufferUtil.serializedSizeWithShortLength(expression.column().name.bytes, sizes) + + expression.operator.serializedSize(); + + switch (expression.kind()) + { + case SIMPLE: + size += ByteBufferUtil.serializedSizeWithShortLength(((SimpleExpression)expression).value, sizes); + break; + case MAP_EQUALITY: + MapEqualityExpression mexpr = (MapEqualityExpression)expression; + if (version < MessagingService.VERSION_30) + size += ByteBufferUtil.serializedSizeWithShortLength(mexpr.getIndexValue(), sizes); + else + size += ByteBufferUtil.serializedSizeWithShortLength(mexpr.key, sizes) + + ByteBufferUtil.serializedSizeWithShortLength(mexpr.value, sizes); + break; + case THRIFT_DYN_EXPR: + size += ByteBufferUtil.serializedSizeWithShortLength(((ThriftExpression)expression).value, sizes); + break; + } + return size; + } + } + } + + /** + * An expression of the form 'column' 'op' 'value'. + */ + private static class SimpleExpression extends Expression + { + public SimpleExpression(ColumnDefinition column, Operator operator, ByteBuffer value) + { + super(column, operator, value); + } + + public boolean isSatisfiedBy(DecoratedKey partitionKey, Row row) + { + // We support null conditions for LWT (in ColumnCondition) but not for RowFilter. + // TODO: we should try to merge both code someday. + assert value != null; + + if (row.isStatic() != column.isStatic()) + return true; + + switch (operator) + { + case EQ: + case LT: + case LTE: + case GTE: + case GT: + case NEQ: + { + assert !column.isComplex() : "Only CONTAINS and CONTAINS_KEY are supported for 'complex' types"; + ByteBuffer foundValue = getValue(partitionKey, row); + // Note that CQL expression are always of the form 'x < 4', i.e. the tested value is on the left. + return foundValue != null && operator.isSatisfiedBy(column.type, foundValue, value); + } + case CONTAINS: + assert column.type.isCollection(); + CollectionType type = (CollectionType)column.type; + if (column.isComplex()) + { + Iterator iter = row.getCells(column); + while (iter.hasNext()) + { + Cell cell = iter.next(); + if (type.kind == CollectionType.Kind.SET) + { + if (type.nameComparator().compare(cell.path().get(0), value) == 0) + return true; + } + else + { + if (type.valueComparator().compare(cell.value(), value) == 0) + return true; + } + } + return false; + } + else + { + ByteBuffer foundValue = getValue(partitionKey, row); + if (foundValue == null) + return false; + + switch (type.kind) + { + case LIST: + ListType listType = (ListType)type; + return listType.compose(foundValue).contains(listType.getElementsType().compose(value)); + case SET: + SetType setType = (SetType)type; + return setType.compose(foundValue).contains(setType.getElementsType().compose(value)); + case MAP: + MapType mapType = (MapType)type; + return mapType.compose(foundValue).containsValue(mapType.getValuesType().compose(value)); + } + throw new AssertionError(); + } + case CONTAINS_KEY: + assert column.type.isCollection() && column.type instanceof MapType; + MapType mapType = (MapType)column.type; + if (column.isComplex()) + { + return row.getCell(column, CellPath.create(value)) != null; + } + else + { + ByteBuffer foundValue = getValue(partitionKey, row); + return foundValue != null && mapType.getSerializer().getSerializedValue(foundValue, value, mapType.getKeysType()) != null; + } + + case IN: + // It wouldn't be terribly hard to support this (though doing so would imply supporting + // IN for 2ndary index) but currently we don't. + throw new AssertionError(); + } + throw new AssertionError(); + } + + @Override + public String toString() + { + AbstractType type = column.type; + switch (operator) + { + case CONTAINS: + assert type instanceof CollectionType; + CollectionType ct = (CollectionType)type; + type = ct.kind == CollectionType.Kind.SET ? ct.nameComparator() : ct.valueComparator(); + break; + case CONTAINS_KEY: + assert type instanceof MapType; + type = ((MapType)type).nameComparator(); + break; + case IN: + type = ListType.getInstance(type, false); + break; + default: + break; + } + return String.format("%s %s %s", column.name, operator, type.getString(value)); + } + + @Override + Kind kind() + { + return Kind.SIMPLE; + } + } + + /** + * An expression of the form 'column' ['key'] = 'value' (which is only + * supported when 'column' is a map). + */ + private static class MapEqualityExpression extends Expression + { + private final ByteBuffer key; + + public MapEqualityExpression(ColumnDefinition column, ByteBuffer key, Operator operator, ByteBuffer value) + { + super(column, operator, value); + assert column.type instanceof MapType && operator == Operator.EQ; + this.key = key; + } + + @Override + public void validateForIndexing() throws InvalidRequestException + { + super.validateForIndexing(); + checkNotNull(key, "Unsupported null value for key of map column %s", column.name); + checkBindValueSet(key, "Unsupported unset value for key of map column %s", column.name); + } + + @Override + public ByteBuffer getIndexValue() + { + return CompositeType.build(key, value); + } + + public boolean isSatisfiedBy(DecoratedKey partitionKey, Row row) + { + assert key != null; + // We support null conditions for LWT (in ColumnCondition) but not for RowFilter. + // TODO: we should try to merge both code someday. + assert value != null; + + if (row.isStatic() != column.isStatic()) + return true; + + MapType mt = (MapType)column.type; + if (column.isComplex()) + { + Cell cell = row.getCell(column, CellPath.create(key)); + return cell != null && mt.valueComparator().compare(cell.value(), value) == 0; + } + else + { + ByteBuffer serializedMap = getValue(partitionKey, row); + if (serializedMap == null) + return false; + + ByteBuffer foundValue = mt.getSerializer().getSerializedValue(serializedMap, key, mt.getKeysType()); + return foundValue != null && mt.valueComparator().compare(foundValue, value) == 0; + } + } + + @Override + public String toString() + { + MapType mt = (MapType)column.type; + return String.format("%s[%s] = %s", column.name, mt.nameComparator().getString(key), mt.valueComparator().getString(value)); + } + + @Override + public boolean equals(Object o) + { + if (this == o) + return true; + + if (!(o instanceof MapEqualityExpression)) + return false; + + MapEqualityExpression that = (MapEqualityExpression)o; + + return Objects.equal(this.column.name, that.column.name) + && Objects.equal(this.operator, that.operator) + && Objects.equal(this.key, that.key) + && Objects.equal(this.value, that.value); + } + + @Override + public int hashCode() + { + return Objects.hashCode(column.name, operator, key, value); + } + + @Override + Kind kind() + { + return Kind.MAP_EQUALITY; + } + } + + /** + * An expression of the form 'name' = 'value', but where 'name' is actually the + * clustering value for a compact table. This is only for thrift. + */ + private static class ThriftExpression extends Expression + { + private final CFMetaData metadata; + + public ThriftExpression(CFMetaData metadata, ByteBuffer name, Operator operator, ByteBuffer value) + { + super(makeDefinition(metadata, name), operator, value); + assert metadata.isCompactTable(); + this.metadata = metadata; + } + + private static ColumnDefinition makeDefinition(CFMetaData metadata, ByteBuffer name) + { + ColumnDefinition def = metadata.getColumnDefinition(name); + if (def != null) + return def; + + // In thrift, we actually allow expression on non-defined columns for the sake of filtering. To accomodate + // this we create a "fake" definition. This is messy but it works so is probably good enough. + return ColumnDefinition.regularDef(metadata, name, metadata.compactValueColumn().type, null); + } + + public boolean isSatisfiedBy(DecoratedKey partitionKey, Row row) + { + assert value != null; + + // On thrift queries, even if the column expression is a "static" one, we'll have convert it as a "dynamic" + // one in ThriftResultsMerger, so we always expect it to be a dynamic one. Further, we expect this is only + // called when the row clustering does match the column (see ThriftFilter above). + assert row.clustering().equals(makeCompactClustering(metadata, column.name.bytes)); + Cell cell = row.getCell(metadata.compactValueColumn()); + return cell != null && operator.isSatisfiedBy(column.type, cell.value(), value); + } + + @Override + public String toString() + { + return String.format("%s %s %s", column.name, operator, column.type.getString(value)); + } + + @Override + Kind kind() + { + return Kind.THRIFT_DYN_EXPR; + } + } + + public static class Serializer + { + public void serialize(RowFilter filter, DataOutputPlus out, int version) throws IOException + { + out.writeBoolean(filter instanceof ThriftFilter); + out.writeShort(filter.expressions.size()); + for (Expression expr : filter.expressions) + Expression.serializer.serialize(expr, out, version); + } + + public RowFilter deserialize(DataInput in, int version, CFMetaData metadata) throws IOException + { + boolean forThrift = in.readBoolean(); + int size = in.readUnsignedShort(); + List expressions = new ArrayList<>(size); + for (int i = 0; i < size; i++) + expressions.add(Expression.serializer.deserialize(in, version, metadata)); + return forThrift + ? new ThriftFilter(expressions) + : new CQLFilter(expressions); + } + + public long serializedSize(RowFilter filter, int version) + { + TypeSizes sizes = TypeSizes.NATIVE; + long size = 1 // forThrift + + sizes.sizeof((short)filter.expressions.size()); + for (Expression expr : filter.expressions) + size += Expression.serializer.serializedSize(expr, version); + return size; + } + } +} diff --git a/src/java/org/apache/cassandra/db/filter/SliceQueryFilter.java b/src/java/org/apache/cassandra/db/filter/SliceQueryFilter.java deleted file mode 100644 index 45711615be..0000000000 --- a/src/java/org/apache/cassandra/db/filter/SliceQueryFilter.java +++ /dev/null @@ -1,583 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.apache.cassandra.db.filter; - -import java.nio.ByteBuffer; -import java.io.DataInput; -import java.io.IOException; -import java.util.*; - -import com.google.common.collect.AbstractIterator; -import com.google.common.collect.Iterators; -import org.apache.cassandra.config.CFMetaData; -import org.apache.cassandra.db.composites.*; -import org.apache.cassandra.utils.ByteBufferUtil; -import org.apache.cassandra.utils.Pair; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; - -import org.apache.cassandra.config.DatabaseDescriptor; -import org.apache.cassandra.db.*; -import org.apache.cassandra.db.columniterator.OnDiskAtomIterator; -import org.apache.cassandra.io.IVersionedSerializer; -import org.apache.cassandra.io.sstable.format.SSTableReader; -import org.apache.cassandra.io.util.DataOutputPlus; -import org.apache.cassandra.io.util.FileDataInput; -import org.apache.cassandra.service.ClientWarn; -import org.apache.cassandra.tracing.Tracing; - -public class SliceQueryFilter implements IDiskAtomFilter -{ - private static final Logger logger = LoggerFactory.getLogger(SliceQueryFilter.class); - - /** - * A special value for compositesToGroup that indicates that partitioned tombstones should not be included in results - * or count towards the limit. See CASSANDRA-8490 for more details on why this is needed (and done this way). - **/ - public static final int IGNORE_TOMBSTONED_PARTITIONS = -2; - - public final ColumnSlice[] slices; - public final boolean reversed; - public volatile int count; - public final int compositesToGroup; - - // Not serialized, just a ack for range slices to find the number of live column counted, even when we group - private ColumnCounter columnCounter; - - public SliceQueryFilter(Composite start, Composite finish, boolean reversed, int count) - { - this(new ColumnSlice(start, finish), reversed, count); - } - - public SliceQueryFilter(Composite start, Composite finish, boolean reversed, int count, int compositesToGroup) - { - this(new ColumnSlice(start, finish), reversed, count, compositesToGroup); - } - - public SliceQueryFilter(ColumnSlice slice, boolean reversed, int count) - { - this(new ColumnSlice[]{ slice }, reversed, count); - } - - public SliceQueryFilter(ColumnSlice slice, boolean reversed, int count, int compositesToGroup) - { - this(new ColumnSlice[]{ slice }, reversed, count, compositesToGroup); - } - - /** - * Constructor that accepts multiple slices. All slices are assumed to be in the same direction (forward or - * reversed). - */ - public SliceQueryFilter(ColumnSlice[] slices, boolean reversed, int count) - { - this(slices, reversed, count, -1); - } - - public SliceQueryFilter(ColumnSlice[] slices, boolean reversed, int count, int compositesToGroup) - { - this.slices = slices; - this.reversed = reversed; - this.count = count; - this.compositesToGroup = compositesToGroup; - } - - public SliceQueryFilter cloneShallow() - { - return new SliceQueryFilter(slices, reversed, count, compositesToGroup); - } - - public SliceQueryFilter withUpdatedCount(int newCount) - { - return new SliceQueryFilter(slices, reversed, newCount, compositesToGroup); - } - - public SliceQueryFilter withUpdatedSlices(ColumnSlice[] newSlices) - { - return new SliceQueryFilter(newSlices, reversed, count, compositesToGroup); - } - - /** Returns true if the slice includes static columns, false otherwise. */ - private boolean sliceIncludesStatics(ColumnSlice slice, CFMetaData cfm) - { - return cfm.hasStaticColumns() && - slice.includes(reversed ? cfm.comparator.reverseComparator() : cfm.comparator, cfm.comparator.staticPrefix().end()); - } - - public boolean hasStaticSlice(CFMetaData cfm) - { - for (ColumnSlice slice : slices) - if (sliceIncludesStatics(slice, cfm)) - return true; - - return false; - } - - /** - * Splits this filter into two SliceQueryFilters: one that slices only the static columns, and one that slices the - * remainder of the normal data. - * - * This should only be called when the filter is reversed and the filter is known to cover static columns (through - * hasStaticSlice()). - * - * @return a pair of (static, normal) SliceQueryFilters - */ - public Pair splitOutStaticSlice(CFMetaData cfm) - { - assert reversed; - - Composite staticSliceEnd = cfm.comparator.staticPrefix().end(); - List nonStaticSlices = new ArrayList<>(slices.length); - for (ColumnSlice slice : slices) - { - if (sliceIncludesStatics(slice, cfm)) - nonStaticSlices.add(new ColumnSlice(slice.start, staticSliceEnd)); - else - nonStaticSlices.add(slice); - } - - return Pair.create( - new SliceQueryFilter(staticSliceEnd, Composites.EMPTY, true, count, compositesToGroup), - new SliceQueryFilter(nonStaticSlices.toArray(new ColumnSlice[nonStaticSlices.size()]), true, count, compositesToGroup)); - } - - public SliceQueryFilter withUpdatedStart(Composite newStart, CFMetaData cfm) - { - Comparator cmp = reversed ? cfm.comparator.reverseComparator() : cfm.comparator; - - // Check our slices to see if any fall before the new start (in which case they can be removed) or - // if they contain the new start (in which case they should start from the page start). However, if the - // slices would include static columns, we need to ensure they are also fetched, and so a separate - // slice for the static columns may be required. - // Note that if the query is reversed, we can't handle statics by simply adding a separate slice here, so - // the reversed case is handled by SliceFromReadCommand instead. See CASSANDRA-8502 for more details. - List newSlices = new ArrayList<>(); - boolean pastNewStart = false; - for (ColumnSlice slice : slices) - { - if (pastNewStart) - { - newSlices.add(slice); - continue; - } - - if (slice.isBefore(cmp, newStart)) - { - if (!reversed && sliceIncludesStatics(slice, cfm)) - newSlices.add(new ColumnSlice(Composites.EMPTY, cfm.comparator.staticPrefix().end())); - - continue; - } - else if (slice.includes(cmp, newStart)) - { - if (!reversed && sliceIncludesStatics(slice, cfm) && !newStart.isEmpty()) - newSlices.add(new ColumnSlice(Composites.EMPTY, cfm.comparator.staticPrefix().end())); - - newSlices.add(new ColumnSlice(newStart, slice.finish)); - } - else - { - newSlices.add(slice); - } - - pastNewStart = true; - } - return withUpdatedSlices(newSlices.toArray(new ColumnSlice[newSlices.size()])); - } - - public Iterator getColumnIterator(ColumnFamily cf) - { - assert cf != null; - return reversed ? cf.reverseIterator(slices) : cf.iterator(slices); - } - - public OnDiskAtomIterator getColumnIterator(final DecoratedKey key, final ColumnFamily cf) - { - assert cf != null; - final Iterator iter = getColumnIterator(cf); - - return new OnDiskAtomIterator() - { - public ColumnFamily getColumnFamily() - { - return cf; - } - - public DecoratedKey getKey() - { - return key; - } - - public boolean hasNext() - { - return iter.hasNext(); - } - - public OnDiskAtom next() - { - return iter.next(); - } - - public void close() throws IOException { } - - public void remove() - { - throw new UnsupportedOperationException(); - } - }; - } - - public OnDiskAtomIterator getSSTableColumnIterator(SSTableReader sstable, DecoratedKey key) - { - return sstable.iterator(key, slices, reversed); - } - - public OnDiskAtomIterator getSSTableColumnIterator(SSTableReader sstable, FileDataInput file, DecoratedKey key, RowIndexEntry indexEntry) - { - return sstable.iterator(file, key, slices, reversed, indexEntry); - } - - public Comparator getColumnComparator(CellNameType comparator) - { - return reversed ? comparator.columnReverseComparator() : comparator.columnComparator(false); - } - - public void collectReducedColumns(ColumnFamily container, Iterator reducedColumns, DecoratedKey key, int gcBefore, long now) - { - columnCounter = columnCounter(container.getComparator(), now); - DeletionInfo.InOrderTester tester = container.deletionInfo().inOrderTester(reversed); - - while (reducedColumns.hasNext()) - { - Cell cell = reducedColumns.next(); - - if (logger.isTraceEnabled()) - logger.trace("collecting {} of {}: {}", columnCounter.live(), count, cell.getString(container.getComparator())); - - // An expired tombstone will be immediately discarded in memory, and needn't be counted. - // Neither should be any cell shadowed by a range- or a partition tombstone. - if (cell.getLocalDeletionTime() < gcBefore || !columnCounter.count(cell, tester)) - continue; - - if (columnCounter.live() > count) - break; - - if (respectTombstoneThresholds() && columnCounter.tombstones() > DatabaseDescriptor.getTombstoneFailureThreshold()) - { - Tracing.trace("Scanned over {} tombstones; query aborted (see tombstone_failure_threshold); slices={}", - DatabaseDescriptor.getTombstoneFailureThreshold(), getSlicesInfo(container)); - - throw new TombstoneOverwhelmingException(columnCounter.tombstones(), - count, - container.metadata().ksName, - container.metadata().cfName, - container.getComparator().getString(cell.name()), - getSlicesInfo(container)); - } - - container.appendColumn(cell); - } - - boolean warnTombstones = logger.isWarnEnabled() && respectTombstoneThresholds() && columnCounter.tombstones() > DatabaseDescriptor.getTombstoneWarnThreshold(); - if (warnTombstones) - { - String msg = String.format("Read %d live and %d tombstone cells in %s.%s for key: %1.512s (see tombstone_warn_threshold). %d columns were requested, slices=%1.512s", - columnCounter.live(), - columnCounter.tombstones(), - container.metadata().ksName, - container.metadata().cfName, - container.metadata().getKeyValidator().getString(key.getKey()), - count, - getSlicesInfo(container)); - ClientWarn.warn(msg); - logger.warn(msg); - } - Tracing.trace("Read {} live and {} tombstone cells{}", - columnCounter.live(), - columnCounter.tombstones(), - warnTombstones ? " (see tombstone_warn_threshold)" : ""); - } - - private String getSlicesInfo(ColumnFamily container) - { - StringBuilder sb = new StringBuilder(); - CellNameType type = container.metadata().comparator; - for (ColumnSlice sl : slices) - { - assert sl != null; - - sb.append('['); - sb.append(type.getString(sl.start)); - sb.append('-'); - sb.append(type.getString(sl.finish)); - sb.append(']'); - } - return sb.toString(); - } - - protected boolean respectTombstoneThresholds() - { - return true; - } - - public int getLiveCount(ColumnFamily cf, long now) - { - return columnCounter(cf.getComparator(), now).countAll(cf).live(); - } - - public ColumnCounter columnCounter(CellNameType comparator, long now) - { - if (compositesToGroup < 0) - return new ColumnCounter(now); - else if (compositesToGroup == 0) - return new ColumnCounter.GroupByPrefix(now, null, 0); - else if (reversed) - return new ColumnCounter.GroupByPrefixReversed(now, comparator, compositesToGroup); - else - return new ColumnCounter.GroupByPrefix(now, comparator, compositesToGroup); - } - - public void trim(ColumnFamily cf, int trimTo, long now) - { - // each cell can increment the count by at most one, so if we have fewer cells than trimTo, we can skip trimming - if (cf.getColumnCount() < trimTo) - return; - - ColumnCounter counter = columnCounter(cf.getComparator(), now); - - Collection cells = reversed - ? cf.getReverseSortedColumns() - : cf.getSortedColumns(); - - DeletionInfo.InOrderTester tester = cf.deletionInfo().inOrderTester(reversed); - - for (Iterator iter = cells.iterator(); iter.hasNext(); ) - { - Cell cell = iter.next(); - counter.count(cell, tester); - - if (counter.live() > trimTo) - { - iter.remove(); - while (iter.hasNext()) - { - iter.next(); - iter.remove(); - } - } - } - } - - public Composite start() - { - return this.slices[0].start; - } - - public Composite finish() - { - return this.slices[slices.length - 1].finish; - } - - public void setStart(Composite start) - { - assert slices.length == 1; - this.slices[0] = new ColumnSlice(start, this.slices[0].finish); - } - - public int lastCounted() - { - // If we have a slice limit set, columnCounter.live() can overcount by one because we have to call - // columnCounter.count() before we can tell if we've exceeded the slice limit (and accordingly, should not - // add the cells to returned container). To deal with this overcounting, we take the min of the slice - // limit and the counter's count. - return columnCounter == null ? 0 : Math.min(columnCounter.live(), count); - } - - public int lastTombstones() - { - return columnCounter == null ? 0 : columnCounter.tombstones(); - } - - public int lastLive() - { - return columnCounter == null ? 0 : columnCounter.live(); - } - - @Override - public String toString() - { - return "SliceQueryFilter [reversed=" + reversed + ", slices=" + Arrays.toString(slices) + ", count=" + count + ", toGroup = " + compositesToGroup + "]"; - } - - public boolean isReversed() - { - return reversed; - } - - public void updateColumnsLimit(int newLimit) - { - count = newLimit; - } - - public boolean maySelectPrefix(CType type, Composite prefix) - { - for (ColumnSlice slice : slices) - if (slice.includes(type, prefix)) - return true; - return false; - } - - public boolean shouldInclude(SSTableReader sstable) - { - List minColumnNames = sstable.getSSTableMetadata().minColumnNames; - List maxColumnNames = sstable.getSSTableMetadata().maxColumnNames; - CellNameType comparator = sstable.metadata.comparator; - - if (minColumnNames.isEmpty() || maxColumnNames.isEmpty()) - return true; - - for (ColumnSlice slice : slices) - if (slice.intersects(minColumnNames, maxColumnNames, comparator, reversed)) - return true; - - return false; - } - - public boolean isHeadFilter() - { - return slices.length == 1 && slices[0].start.isEmpty() && !reversed; - } - - public boolean countCQL3Rows(CellNameType comparator) - { - // If comparator is dense a cell == a CQL3 rows so we're always counting CQL3 rows - // in particular. Otherwise, we do so only if we group the cells into CQL rows. - return comparator.isDense() || compositesToGroup >= 0; - } - - public boolean isFullyCoveredBy(ColumnFamily cf, long now) - { - // cf is the beginning of a partition. It covers this filter if: - // 1) either this filter requests the head of the partition and request less - // than what cf has to offer (note: we do need to use getLiveCount() for that - // as it knows if the filter count cells or CQL3 rows). - // 2) the start and finish bound of this filter are included in cf. - if (isHeadFilter() && count <= getLiveCount(cf, now)) - return true; - - if (start().isEmpty() || finish().isEmpty() || !cf.hasColumns()) - return false; - - Composite low = isReversed() ? finish() : start(); - Composite high = isReversed() ? start() : finish(); - - CellName first = cf.iterator(ColumnSlice.ALL_COLUMNS_ARRAY).next().name(); - CellName last = cf.reverseIterator(ColumnSlice.ALL_COLUMNS_ARRAY).next().name(); - - return cf.getComparator().compare(first, low) <= 0 - && cf.getComparator().compare(high, last) <= 0; - } - - public static class Serializer implements IVersionedSerializer - { - private CType type; - - public Serializer(CType type) - { - this.type = type; - } - - public void serialize(SliceQueryFilter f, DataOutputPlus out, int version) throws IOException - { - out.writeInt(f.slices.length); - for (ColumnSlice slice : f.slices) - type.sliceSerializer().serialize(slice, out, version); - out.writeBoolean(f.reversed); - int count = f.count; - out.writeInt(count); - - out.writeInt(f.compositesToGroup); - } - - public SliceQueryFilter deserialize(DataInput in, int version) throws IOException - { - ColumnSlice[] slices; - slices = new ColumnSlice[in.readInt()]; - for (int i = 0; i < slices.length; i++) - slices[i] = type.sliceSerializer().deserialize(in, version); - boolean reversed = in.readBoolean(); - int count = in.readInt(); - int compositesToGroup = in.readInt(); - - return new SliceQueryFilter(slices, reversed, count, compositesToGroup); - } - - public long serializedSize(SliceQueryFilter f, int version) - { - TypeSizes sizes = TypeSizes.NATIVE; - - int size = 0; - size += sizes.sizeof(f.slices.length); - for (ColumnSlice slice : f.slices) - size += type.sliceSerializer().serializedSize(slice, version); - size += sizes.sizeof(f.reversed); - size += sizes.sizeof(f.count); - - size += sizes.sizeof(f.compositesToGroup); - return size; - } - } - - public Iterator getRangeTombstoneIterator(final ColumnFamily source) - { - final DeletionInfo delInfo = source.deletionInfo(); - if (!delInfo.hasRanges() || slices.length == 0) - return Iterators.emptyIterator(); - - return new AbstractIterator() - { - private int sliceIdx = 0; - private Iterator sliceIter = currentRangeIter(); - - protected RangeTombstone computeNext() - { - while (true) - { - if (sliceIter.hasNext()) - return sliceIter.next(); - - if (!nextSlice()) - return endOfData(); - - sliceIter = currentRangeIter(); - } - } - - private Iterator currentRangeIter() - { - ColumnSlice slice = slices[reversed ? (slices.length - 1 - sliceIdx) : sliceIdx]; - return reversed ? delInfo.rangeIterator(slice.finish, slice.start) - : delInfo.rangeIterator(slice.start, slice.finish); - } - - private boolean nextSlice() - { - return ++sliceIdx < slices.length; - } - }; - } -} diff --git a/src/java/org/apache/cassandra/db/filter/TombstoneOverwhelmingException.java b/src/java/org/apache/cassandra/db/filter/TombstoneOverwhelmingException.java index 7624e1bd15..98b539e458 100644 --- a/src/java/org/apache/cassandra/db/filter/TombstoneOverwhelmingException.java +++ b/src/java/org/apache/cassandra/db/filter/TombstoneOverwhelmingException.java @@ -18,49 +18,51 @@ */ package org.apache.cassandra.db.filter; -import org.apache.cassandra.db.DecoratedKey; +import java.nio.ByteBuffer; + +import org.apache.cassandra.config.CFMetaData; +import org.apache.cassandra.db.*; +import org.apache.cassandra.db.marshal.*; public class TombstoneOverwhelmingException extends RuntimeException { - private final int numTombstones; - private final int numRequested; - private final String ksName; - private final String cfName; - private final String lastCellName; - private final String slicesInfo; - private String partitionKey = null; - - public TombstoneOverwhelmingException(int numTombstones, - int numRequested, - String ksName, - String cfName, - String lastCellName, - String slicesInfo) + public TombstoneOverwhelmingException(int numTombstones, String query, CFMetaData metadata, DecoratedKey lastPartitionKey, ClusteringPrefix lastClustering) { - this.numTombstones = numTombstones; - this.numRequested = numRequested; - this.ksName = ksName; - this.cfName = cfName; - this.lastCellName = lastCellName; - this.slicesInfo = slicesInfo; + super(String.format("Scanned over %d tombstones during query '%s' (last scanned row partion key was (%s)); query aborted", + numTombstones, query, makePKString(metadata, lastPartitionKey.getKey(), lastClustering))); } - public void setKey(DecoratedKey key) + private static String makePKString(CFMetaData metadata, ByteBuffer partitionKey, ClusteringPrefix clustering) { - if (key != null) - partitionKey = key.toString(); - } + StringBuilder sb = new StringBuilder(); - public String getLocalizedMessage() - { - return getMessage(); - } + if (clustering.size() > 0) + sb.append("("); - public String getMessage() - { - return String.format( - "Scanned over %d tombstones in %s.%s; %d columns were requested; query aborted " + - "(see tombstone_failure_threshold); partitionKey=%s; lastCell=%s; slices=%s", - numTombstones, ksName, cfName, numRequested, partitionKey, lastCellName, slicesInfo); + // TODO: We should probably make that a lot easier/transparent for partition keys + AbstractType pkType = metadata.getKeyValidator(); + if (pkType instanceof CompositeType) + { + CompositeType ct = (CompositeType)pkType; + ByteBuffer[] values = ct.split(partitionKey); + for (int i = 0; i < values.length; i++) + { + if (i > 0) + sb.append(", "); + sb.append(ct.types.get(i).getString(values[i])); + } + } + else + { + sb.append(pkType.getString(partitionKey)); + } + + if (clustering.size() > 0) + sb.append(")"); + + for (int i = 0; i < clustering.size(); i++) + sb.append(", ").append(metadata.comparator.subtype(i).getString(clustering.get(i))); + + return sb.toString(); } } diff --git a/src/java/org/apache/cassandra/db/index/AbstractSimplePerColumnSecondaryIndex.java b/src/java/org/apache/cassandra/db/index/AbstractSimplePerColumnSecondaryIndex.java index ba48350806..0b6577e1cd 100644 --- a/src/java/org/apache/cassandra/db/index/AbstractSimplePerColumnSecondaryIndex.java +++ b/src/java/org/apache/cassandra/db/index/AbstractSimplePerColumnSecondaryIndex.java @@ -23,11 +23,11 @@ import java.util.concurrent.Future; import org.apache.cassandra.config.CFMetaData; import org.apache.cassandra.config.ColumnDefinition; import org.apache.cassandra.db.*; -import org.apache.cassandra.db.composites.CellName; -import org.apache.cassandra.db.composites.CellNameType; +import org.apache.cassandra.db.rows.*; +import org.apache.cassandra.db.partitions.*; import org.apache.cassandra.db.marshal.AbstractType; import org.apache.cassandra.dht.LocalPartitioner; -import org.apache.cassandra.utils.ByteBufferUtil; +import org.apache.cassandra.exceptions.InvalidRequestException; import org.apache.cassandra.utils.FBUtilities; import org.apache.cassandra.utils.concurrent.OpOrder; @@ -51,8 +51,7 @@ public abstract class AbstractSimplePerColumnSecondaryIndex extends PerColumnSec columnDef = columnDefs.iterator().next(); - CellNameType indexComparator = SecondaryIndex.getIndexComparator(baseCfs.metadata, columnDef); - CFMetaData indexedCfMetadata = CFMetaData.newIndexMetadata(baseCfs.metadata, columnDef, indexComparator); + CFMetaData indexedCfMetadata = SecondaryIndex.newIndexMetadata(baseCfs.metadata, columnDef); indexCfs = ColumnFamilyStore.createColumnFamilyStore(baseCfs.keyspace, indexedCfMetadata.cfName, new LocalPartitioner(getIndexKeyComparator()), @@ -65,73 +64,98 @@ public abstract class AbstractSimplePerColumnSecondaryIndex extends PerColumnSec return columnDef.type; } + public ColumnDefinition indexedColumn() + { + return columnDef; + } + @Override String indexTypeForGrouping() { return "_internal_"; } - protected abstract CellName makeIndexColumnName(ByteBuffer rowKey, Cell cell); - - protected abstract ByteBuffer getIndexedValue(ByteBuffer rowKey, Cell cell); - - protected abstract AbstractType getExpressionComparator(); - - public String expressionString(IndexExpression expr) + protected Clustering makeIndexClustering(ByteBuffer rowKey, Clustering clustering, Cell cell) { - return String.format("'%s.%s %s %s'", - baseCfs.name, - getExpressionComparator().getString(expr.column), - expr.operator, - baseCfs.metadata.getColumnDefinition(expr.column).type.getString(expr.value)); + return makeIndexClustering(rowKey, clustering, cell == null ? null : cell.path()); } - public void delete(ByteBuffer rowKey, Cell cell, OpOrder.Group opGroup) + protected Clustering makeIndexClustering(ByteBuffer rowKey, Clustering clustering, CellPath path) { - deleteForCleanup(rowKey, cell, opGroup); + return buildIndexClusteringPrefix(rowKey, clustering, path).build(); } - public void deleteForCleanup(ByteBuffer rowKey, Cell cell, OpOrder.Group opGroup) + protected Slice.Bound makeIndexBound(ByteBuffer rowKey, Slice.Bound bound) { - if (!cell.isLive()) - return; + return buildIndexClusteringPrefix(rowKey, bound, null).buildBound(bound.isStart(), bound.isInclusive()); + } - DecoratedKey valueKey = getIndexKeyFor(getIndexedValue(rowKey, cell)); - int localDeletionTime = (int) (System.currentTimeMillis() / 1000); - ColumnFamily cfi = ArrayBackedSortedColumns.factory.create(indexCfs.metadata, false, 1); - cfi.addTombstone(makeIndexColumnName(rowKey, cell), localDeletionTime, cell.timestamp()); - indexCfs.apply(valueKey, cfi, SecondaryIndexManager.nullUpdater, opGroup, null); + protected abstract CBuilder buildIndexClusteringPrefix(ByteBuffer rowKey, ClusteringPrefix prefix, CellPath path); + + protected ByteBuffer getIndexedValue(ByteBuffer rowKey, Clustering clustering, Cell cell) + { + return cell == null + ? getIndexedValue(rowKey, clustering, null, null) + : getIndexedValue(rowKey, clustering, cell.value(), cell.path()); + } + + protected abstract ByteBuffer getIndexedValue(ByteBuffer rowKey, Clustering clustering, ByteBuffer cellValue, CellPath cellPath); + + public void delete(ByteBuffer rowKey, Clustering clustering, Cell cell, OpOrder.Group opGroup, int nowInSec) + { + deleteForCleanup(rowKey, clustering, cell, opGroup, nowInSec); + } + + public void deleteForCleanup(ByteBuffer rowKey, Clustering clustering, Cell cell, OpOrder.Group opGroup, int nowInSec) + { + delete(rowKey, clustering, cell.value(), cell.path(), new SimpleDeletionTime(cell.livenessInfo().timestamp(), nowInSec), opGroup); + } + + public void delete(ByteBuffer rowKey, Clustering clustering, ByteBuffer cellValue, CellPath path, DeletionTime deletion, OpOrder.Group opGroup) + { + DecoratedKey valueKey = getIndexKeyFor(getIndexedValue(rowKey, clustering, cellValue, path)); + PartitionUpdate upd = new PartitionUpdate(indexCfs.metadata, valueKey, PartitionColumns.NONE, 1); + Row.Writer writer = upd.writer(); + Rows.writeClustering(makeIndexClustering(rowKey, clustering, path), writer); + writer.writeRowDeletion(deletion); + writer.endOfRow(); + indexCfs.apply(upd, SecondaryIndexManager.nullUpdater, opGroup, null); if (logger.isDebugEnabled()) - logger.debug("removed index entry for cleaned-up value {}:{}", valueKey, cfi); + logger.debug("removed index entry for cleaned-up value {}:{}", valueKey, upd); } - public void insert(ByteBuffer rowKey, Cell cell, OpOrder.Group opGroup) + public void insert(ByteBuffer rowKey, Clustering clustering, Cell cell, OpOrder.Group opGroup) { - DecoratedKey valueKey = getIndexKeyFor(getIndexedValue(rowKey, cell)); - ColumnFamily cfi = ArrayBackedSortedColumns.factory.create(indexCfs.metadata, false, 1); - CellName name = makeIndexColumnName(rowKey, cell); - if (cell instanceof ExpiringCell) - { - ExpiringCell ec = (ExpiringCell) cell; - cfi.addColumn(new BufferExpiringCell(name, ByteBufferUtil.EMPTY_BYTE_BUFFER, ec.timestamp(), ec.getTimeToLive(), ec.getLocalDeletionTime())); - } - else - { - cfi.addColumn(new BufferCell(name, ByteBufferUtil.EMPTY_BYTE_BUFFER, cell.timestamp())); - } - if (logger.isDebugEnabled()) - logger.debug("applying index row {} in {}", indexCfs.metadata.getKeyValidator().getString(valueKey.getKey()), cfi); - - indexCfs.apply(valueKey, cfi, SecondaryIndexManager.nullUpdater, opGroup, null); + insert(rowKey, clustering, cell, cell.livenessInfo(), opGroup); } - public void update(ByteBuffer rowKey, Cell oldCol, Cell col, OpOrder.Group opGroup) + public void insert(ByteBuffer rowKey, Clustering clustering, Cell cell, LivenessInfo info, OpOrder.Group opGroup) + { + DecoratedKey valueKey = getIndexKeyFor(getIndexedValue(rowKey, clustering, cell)); + + PartitionUpdate upd = new PartitionUpdate(indexCfs.metadata, valueKey, PartitionColumns.NONE, 1); + Row.Writer writer = upd.writer(); + Rows.writeClustering(makeIndexClustering(rowKey, clustering, cell), writer); + writer.writePartitionKeyLivenessInfo(info); + writer.endOfRow(); + if (logger.isDebugEnabled()) + logger.debug("applying index row {} in {}", indexCfs.metadata.getKeyValidator().getString(valueKey.getKey()), upd); + + indexCfs.apply(upd, SecondaryIndexManager.nullUpdater, opGroup, null); + } + + public void update(ByteBuffer rowKey, Clustering clustering, Cell oldCell, Cell cell, OpOrder.Group opGroup, int nowInSec) { // insert the new value before removing the old one, so we never have a period - // where the row is invisible to both queries (the opposite seems preferable); see CASSANDRA-5540 - insert(rowKey, col, opGroup); - if (SecondaryIndexManager.shouldCleanupOldValue(oldCol, col)) - delete(rowKey, oldCol, opGroup); + // where the row is invisible to both queries (the opposite seems preferable); see CASSANDRA-5540 + insert(rowKey, clustering, cell, opGroup); + if (SecondaryIndexManager.shouldCleanupOldValue(oldCell, cell)) + delete(rowKey, clustering, oldCell, opGroup, nowInSec); + } + + public boolean indexes(ColumnDefinition column) + { + return column.name.equals(columnDef.name); } public void removeIndex(ByteBuffer columnName) @@ -165,6 +189,12 @@ public abstract class AbstractSimplePerColumnSecondaryIndex extends PerColumnSec return indexCfs; } + protected ClusteringComparator getIndexComparator() + { + assert indexCfs != null; + return indexCfs.metadata.comparator; + } + public String getIndexName() { return indexCfs.name; @@ -172,18 +202,43 @@ public abstract class AbstractSimplePerColumnSecondaryIndex extends PerColumnSec public void reload() { - indexCfs.metadata.reloadSecondaryIndexMetadata(baseCfs.metadata); + indexCfs.metadata.reloadIndexMetadataProperties(baseCfs.metadata); indexCfs.reload(); } - + public long estimateResultRows() { return getIndexCfs().getMeanColumns(); } - public boolean validate(ByteBuffer rowKey, Cell cell) + public void validate(DecoratedKey partitionKey) throws InvalidRequestException { - return getIndexedValue(rowKey, cell).remaining() < FBUtilities.MAX_UNSIGNED_SHORT - && makeIndexColumnName(rowKey, cell).toByteBuffer().remaining() < FBUtilities.MAX_UNSIGNED_SHORT; + if (columnDef.kind == ColumnDefinition.Kind.PARTITION_KEY) + validateIndexedValue(getIndexedValue(partitionKey.getKey(), null, null, null)); + } + + public void validate(Clustering clustering) throws InvalidRequestException + { + if (columnDef.kind == ColumnDefinition.Kind.CLUSTERING_COLUMN) + validateIndexedValue(getIndexedValue(null, clustering, null, null)); + } + + public void validate(ByteBuffer cellValue, CellPath path) throws InvalidRequestException + { + if (!columnDef.isPrimaryKeyColumn()) + validateIndexedValue(getIndexedValue(null, null, cellValue, path)); + } + + private void validateIndexedValue(ByteBuffer value) + { + if (value != null && value.remaining() >= FBUtilities.MAX_UNSIGNED_SHORT) + throw new InvalidRequestException(String.format("Cannot index value of size %d for index %s on %s.%s(%s) (maximum allowed size=%d)", + value.remaining(), getIndexName(), baseKeyspace(), baseTable(), columnDef.name, FBUtilities.MAX_UNSIGNED_SHORT)); + } + + @Override + public String toString() + { + return String.format("%s(%s)", baseTable(), columnDef.name); } } diff --git a/src/java/org/apache/cassandra/db/index/PerColumnSecondaryIndex.java b/src/java/org/apache/cassandra/db/index/PerColumnSecondaryIndex.java index ba902eca13..ab8e688722 100644 --- a/src/java/org/apache/cassandra/db/index/PerColumnSecondaryIndex.java +++ b/src/java/org/apache/cassandra/db/index/PerColumnSecondaryIndex.java @@ -19,9 +19,9 @@ package org.apache.cassandra.db.index; import java.nio.ByteBuffer; +import org.apache.cassandra.db.*; +import org.apache.cassandra.db.rows.*; import org.apache.cassandra.utils.concurrent.OpOrder; -import org.apache.cassandra.db.Cell; -import org.apache.cassandra.utils.FBUtilities; /** * Base class for Secondary indexes that implement a unique index per column @@ -35,12 +35,26 @@ public abstract class PerColumnSecondaryIndex extends SecondaryIndex * @param rowKey the underlying row key which is indexed * @param col all the column info */ - public abstract void delete(ByteBuffer rowKey, Cell col, OpOrder.Group opGroup); + public abstract void delete(ByteBuffer rowKey, Clustering clustering, Cell cell, OpOrder.Group opGroup, int nowInSec); /** * Called when a column has been removed due to a cleanup operation. */ - public abstract void deleteForCleanup(ByteBuffer rowKey, Cell col, OpOrder.Group opGroup); + public abstract void deleteForCleanup(ByteBuffer rowKey, Clustering clustering, Cell cell, OpOrder.Group opGroup, int nowInSec); + + /** + * For indexes on the primary key, index the given PK. + */ + public void maybeIndex(ByteBuffer partitionKey, Clustering clustering, long timestamp, int ttl, OpOrder.Group opGroup, int nowInSec) + { + } + + /** + * For indexes on the primary key, delete the given PK. + */ + public void maybeDelete(ByteBuffer partitionKey, Clustering clustering, DeletionTime deletion, OpOrder.Group opGroup) + { + } /** * insert a column to the index @@ -48,7 +62,7 @@ public abstract class PerColumnSecondaryIndex extends SecondaryIndex * @param rowKey the underlying row key which is indexed * @param col all the column info */ - public abstract void insert(ByteBuffer rowKey, Cell col, OpOrder.Group opGroup); + public abstract void insert(ByteBuffer rowKey, Clustering clustering, Cell cell, OpOrder.Group opGroup); /** * update a column from the index @@ -57,20 +71,44 @@ public abstract class PerColumnSecondaryIndex extends SecondaryIndex * @param oldCol the previous column info * @param col all the column info */ - public abstract void update(ByteBuffer rowKey, Cell oldCol, Cell col, OpOrder.Group opGroup); + public abstract void update(ByteBuffer rowKey, Clustering clustering, Cell oldCell, Cell cell, OpOrder.Group opGroup, int nowInSec); + + protected boolean indexPrimaryKeyColumn() + { + return false; + } + + public void indexRow(DecoratedKey key, Row row, OpOrder.Group opGroup, int nowInSec) + { + Clustering clustering = row.clustering(); + if (indexPrimaryKeyColumn()) + { + // Same as in AtomicBTreePartition.maybeIndexPrimaryKeyColumn + long timestamp = row.primaryKeyLivenessInfo().timestamp(); + int ttl = row.primaryKeyLivenessInfo().ttl(); + + for (Cell cell : row) + { + if (cell.isLive(nowInSec) && cell.livenessInfo().timestamp() > timestamp) + { + timestamp = cell.livenessInfo().timestamp(); + ttl = cell.livenessInfo().ttl(); + } + } + maybeIndex(key.getKey(), clustering, timestamp, ttl, opGroup, nowInSec); + } + for (Cell cell : row) + { + if (!indexes(cell.column())) + continue; + + if (cell.isLive(nowInSec)) + insert(key.getKey(), clustering, cell, opGroup); + } + } public String getNameForSystemKeyspace(ByteBuffer column) { return getIndexName(); } - - public boolean validate(ByteBuffer rowKey, Cell cell) - { - return validate(cell); - } - - public boolean validate(Cell cell) - { - return cell.value().remaining() < FBUtilities.MAX_UNSIGNED_SHORT; - } } diff --git a/src/java/org/apache/cassandra/db/index/PerRowSecondaryIndex.java b/src/java/org/apache/cassandra/db/index/PerRowSecondaryIndex.java index f6f0e8d3f4..502b213f78 100644 --- a/src/java/org/apache/cassandra/db/index/PerRowSecondaryIndex.java +++ b/src/java/org/apache/cassandra/db/index/PerRowSecondaryIndex.java @@ -20,11 +20,9 @@ package org.apache.cassandra.db.index; import java.nio.ByteBuffer; import java.nio.charset.CharacterCodingException; -import org.apache.cassandra.utils.FBUtilities; +import org.apache.cassandra.db.rows.UnfilteredRowIterator; +import org.apache.cassandra.db.partitions.PartitionUpdate; import org.apache.cassandra.utils.concurrent.OpOrder; -import org.apache.cassandra.db.Cell; -import org.apache.cassandra.db.ColumnFamily; -import org.apache.cassandra.db.DecoratedKey; import org.apache.cassandra.utils.ByteBufferUtil; /** @@ -33,19 +31,16 @@ import org.apache.cassandra.utils.ByteBufferUtil; public abstract class PerRowSecondaryIndex extends SecondaryIndex { /** - * Index the given row. - * - * @param rowKey the row key - * @param cf the cf data to be indexed + * Index the given partition. */ - public abstract void index(ByteBuffer rowKey, ColumnFamily cf); + public abstract void index(ByteBuffer key, UnfilteredRowIterator atoms); /** * cleans up deleted columns from cassandra cleanup compaction * * @param key */ - public abstract void delete(DecoratedKey key, OpOrder.Group opGroup); + public abstract void delete(ByteBuffer key, OpOrder.Group opGroup); public String getNameForSystemKeyspace(ByteBuffer columnName) { @@ -58,15 +53,4 @@ public abstract class PerRowSecondaryIndex extends SecondaryIndex throw new RuntimeException(e); } } - - - public boolean validate(ByteBuffer rowKey, Cell cell) - { - return validate(cell); - } - - public boolean validate(Cell cell) - { - return true; - } } diff --git a/src/java/org/apache/cassandra/db/index/SecondaryIndex.java b/src/java/org/apache/cassandra/db/index/SecondaryIndex.java index 11626d68b0..7552fd5884 100644 --- a/src/java/org/apache/cassandra/db/index/SecondaryIndex.java +++ b/src/java/org/apache/cassandra/db/index/SecondaryIndex.java @@ -32,15 +32,11 @@ import org.slf4j.LoggerFactory; import org.apache.cassandra.config.CFMetaData; import org.apache.cassandra.config.ColumnDefinition; +import org.apache.cassandra.config.IndexType; import org.apache.cassandra.cql3.Operator; -import org.apache.cassandra.db.Cell; -import org.apache.cassandra.db.ColumnFamilyStore; -import org.apache.cassandra.db.DecoratedKey; -import org.apache.cassandra.db.SystemKeyspace; +import org.apache.cassandra.db.*; +import org.apache.cassandra.db.rows.*; import org.apache.cassandra.db.compaction.CompactionManager; -import org.apache.cassandra.db.composites.CellName; -import org.apache.cassandra.db.composites.CellNameType; -import org.apache.cassandra.db.composites.SimpleDenseCellNameType; import org.apache.cassandra.db.index.composites.CompositesIndex; import org.apache.cassandra.db.index.keys.KeysIndex; import org.apache.cassandra.db.marshal.AbstractType; @@ -48,6 +44,7 @@ import org.apache.cassandra.db.marshal.BytesType; import org.apache.cassandra.db.marshal.LocalByPartionerType; import org.apache.cassandra.exceptions.ConfigurationException; import org.apache.cassandra.io.sstable.format.SSTableReader; +import org.apache.cassandra.exceptions.InvalidRequestException; import org.apache.cassandra.io.sstable.ReducingKeyIterator; import org.apache.cassandra.service.StorageService; import org.apache.cassandra.utils.FBUtilities; @@ -168,7 +165,7 @@ public abstract class SecondaryIndex * @param columns the list of columns which belong to this index type * @return the secondary index search impl */ - protected abstract SecondaryIndexSearcher createSecondaryIndexSearcher(Set columns); + protected abstract SecondaryIndexSearcher createSecondaryIndexSearcher(Set columns); /** * Forces this indexes' in memory data to disk @@ -308,22 +305,12 @@ public abstract class SecondaryIndex } /** - * Returns true if the provided cell name is indexed by this secondary index. + * Returns true if the provided column is indexed by this secondary index. * * The default implementation checks whether the name is one the columnDef name, * but this should be overriden but subclass if needed. */ - public abstract boolean indexes(CellName name); - - /** - * Returns true if the provided column definition is indexed by this secondary index. - * - * The default implementation checks whether it is contained in this index column definitions set. - */ - public boolean indexes(ColumnDefinition cdef) - { - return columnDefs.contains(cdef); - } + public abstract boolean indexes(ColumnDefinition column); /** * This is the primary way to create a secondary index instance for a CF column. @@ -371,28 +358,45 @@ public abstract class SecondaryIndex return index; } - public abstract boolean validate(ByteBuffer rowKey, Cell cell); + public abstract void validate(DecoratedKey partitionKey) throws InvalidRequestException; + public abstract void validate(Clustering clustering) throws InvalidRequestException; + public abstract void validate(ByteBuffer cellValue, CellPath path) throws InvalidRequestException; public abstract long estimateResultRows(); - /** - * Returns the index comparator for index backed by CFS, or null. - * - * Note: it would be cleaner to have this be a member method. However we need this when opening indexes - * sstables, but by then the CFS won't be fully initiated, so the SecondaryIndex object won't be accessible. - */ - public static CellNameType getIndexComparator(CFMetaData baseMetadata, ColumnDefinition cdef) + protected String baseKeyspace() { - switch (cdef.getIndexType()) + return baseCfs.metadata.ksName; + } + + protected String baseTable() + { + return baseCfs.metadata.cfName; + } + + /** + * Create the index metadata for the index on a given column of a given table. + */ + public static CFMetaData newIndexMetadata(CFMetaData baseMetadata, ColumnDefinition def) + { + if (def.getIndexType() == IndexType.CUSTOM) + return null; + + CFMetaData.Builder builder = CFMetaData.Builder.create(baseMetadata.ksName, baseMetadata.indexColumnFamilyName(def)) + .withId(baseMetadata.cfId) + .addPartitionKey(def.name, def.type); + + if (def.getIndexType() == IndexType.COMPOSITES) { - case KEYS: - return new SimpleDenseCellNameType(keyComparator); - case COMPOSITES: - return CompositesIndex.getIndexComparator(baseMetadata, cdef); - case CUSTOM: - return null; + CompositesIndex.addIndexClusteringColumns(builder, baseMetadata, def); } - throw new AssertionError(); + else + { + assert def.getIndexType() == IndexType.KEYS; + KeysIndex.addIndexClusteringColumns(builder, baseMetadata, def); + } + + return builder.build().reloadIndexMetadataProperties(baseMetadata); } @Override diff --git a/src/java/org/apache/cassandra/db/index/SecondaryIndexBuilder.java b/src/java/org/apache/cassandra/db/index/SecondaryIndexBuilder.java index 916c2861ea..a117f6db0e 100644 --- a/src/java/org/apache/cassandra/db/index/SecondaryIndexBuilder.java +++ b/src/java/org/apache/cassandra/db/index/SecondaryIndexBuilder.java @@ -28,6 +28,7 @@ import org.apache.cassandra.db.compaction.CompactionInfo; import org.apache.cassandra.db.compaction.OperationType; import org.apache.cassandra.db.compaction.CompactionInterruptedException; import org.apache.cassandra.io.sstable.ReducingKeyIterator; +import org.apache.cassandra.utils.FBUtilities; import org.apache.cassandra.utils.UUIDGen; /** @@ -45,7 +46,7 @@ public class SecondaryIndexBuilder extends CompactionInfo.Holder this.cfs = cfs; this.idxNames = idxNames; this.iter = iter; - compactionId = UUIDGen.getTimeUUID(); + this.compactionId = UUIDGen.getTimeUUID(); } public CompactionInfo getCompactionInfo() @@ -59,21 +60,26 @@ public class SecondaryIndexBuilder extends CompactionInfo.Holder public void build() { - while (iter.hasNext()) - { - if (isStopRequested()) - throw new CompactionInterruptedException(getCompactionInfo()); - DecoratedKey key = iter.next(); - Keyspace.indexRow(key, cfs, idxNames); - } - try { - iter.close(); + while (iter.hasNext()) + { + if (isStopRequested()) + throw new CompactionInterruptedException(getCompactionInfo()); + DecoratedKey key = iter.next(); + Keyspace.indexPartition(key, cfs, idxNames); + } } - catch (IOException e) + finally { - throw new RuntimeException(e); + try + { + iter.close(); + } + catch (IOException e) + { + throw new RuntimeException(e); + } } } } diff --git a/src/java/org/apache/cassandra/db/index/SecondaryIndexManager.java b/src/java/org/apache/cassandra/db/index/SecondaryIndexManager.java index 4c1bf45b58..1bd5452103 100644 --- a/src/java/org/apache/cassandra/db/index/SecondaryIndexManager.java +++ b/src/java/org/apache/cassandra/db/index/SecondaryIndexManager.java @@ -18,7 +18,6 @@ package org.apache.cassandra.db.index; import java.nio.ByteBuffer; -import java.nio.charset.CharacterCodingException; import java.util.ArrayList; import java.util.Collection; import java.util.Collections; @@ -40,20 +39,14 @@ import org.slf4j.LoggerFactory; import org.apache.cassandra.config.ColumnDefinition; import org.apache.cassandra.config.IndexType; -import org.apache.cassandra.db.Cell; -import org.apache.cassandra.db.ColumnFamily; -import org.apache.cassandra.db.ColumnFamilyStore; -import org.apache.cassandra.db.DecoratedKey; -import org.apache.cassandra.db.IndexExpression; -import org.apache.cassandra.db.Row; -import org.apache.cassandra.db.SystemKeyspace; +import org.apache.cassandra.db.*; +import org.apache.cassandra.db.rows.*; +import org.apache.cassandra.db.filter.RowFilter; +import org.apache.cassandra.db.partitions.PartitionUpdate; import org.apache.cassandra.db.compaction.CompactionManager; -import org.apache.cassandra.db.composites.CellName; -import org.apache.cassandra.db.filter.ExtendedFilter; import org.apache.cassandra.exceptions.InvalidRequestException; import org.apache.cassandra.io.sstable.ReducingKeyIterator; import org.apache.cassandra.io.sstable.format.SSTableReader; -import org.apache.cassandra.utils.ByteBufferUtil; import org.apache.cassandra.utils.FBUtilities; import org.apache.cassandra.utils.concurrent.OpOrder; @@ -67,11 +60,10 @@ public class SecondaryIndexManager public static final Updater nullUpdater = new Updater() { - public void insert(Cell cell) { } - - public void update(Cell oldCell, Cell cell) { } - - public void remove(Cell current) { } + public void maybeIndex(Clustering clustering, long timestamp, int ttl, DeletionTime deletion) {} + public void insert(Clustering clustering, Cell cell) {} + public void update(Clustering clustering, Cell oldCell, Cell cell) {} + public void remove(Clustering clustering, Cell current) {} public void updateRowLevelIndexes() {} }; @@ -144,6 +136,24 @@ public class SecondaryIndexManager return names; } + public Set perColumnIndexes() + { + Set s = new HashSet<>(); + for (SecondaryIndex index : allIndexes) + if (index instanceof PerColumnSecondaryIndex) + s.add((PerColumnSecondaryIndex)index); + return s; + } + + public Set perRowIndexes() + { + Set s = new HashSet<>(); + for (SecondaryIndex index : allIndexes) + if (index instanceof PerRowSecondaryIndex) + s.add((PerRowSecondaryIndex)index); + return s; + } + /** * Does a full, blocking rebuild of the indexes specified by columns from the sstables. * Does nothing if columns is empty. @@ -171,26 +181,20 @@ public class SecondaryIndexManager logger.info("Index build of {} complete", idxNames); } - public boolean indexes(CellName name, Set indexes) + public boolean indexes(ColumnDefinition column) { - boolean matching = false; - for (SecondaryIndex index : indexes) - { - if (index.indexes(name)) - { - matching = true; - break; - } - } - return matching; + for (SecondaryIndex index : allIndexes) + if (index.indexes(column)) + return true; + return false; } - public Set indexFor(CellName name, Set indexes) + private Set indexFor(ColumnDefinition column) { Set matching = null; - for (SecondaryIndex index : indexes) + for (SecondaryIndex index : allIndexes) { - if (index.indexes(name)) + if (index.indexes(column)) { if (matching == null) matching = new HashSet<>(); @@ -200,36 +204,6 @@ public class SecondaryIndexManager return matching == null ? Collections.emptySet() : matching; } - public boolean indexes(Cell cell) - { - return indexes(cell.name()); - } - - public boolean indexes(CellName name) - { - return indexes(name, allIndexes); - } - - public Set indexFor(CellName name) - { - return indexFor(name, allIndexes); - } - - /** - * @return true if at least one of the indexes can handle the clause. - */ - public boolean hasIndexFor(List clause) - { - if (clause == null || clause.isEmpty()) - return false; - - for (SecondaryIndexSearcher searcher : getIndexSearchersForQuery(clause)) - if (searcher.canHandleIndexClause(clause)) - return true; - - return false; - } - /** * Removes a existing index * @param column the indexed column to remove @@ -325,9 +299,9 @@ public class SecondaryIndexManager * @param column the name of indexes column * @return the index */ - public SecondaryIndex getIndexForColumn(ByteBuffer column) + public SecondaryIndex getIndexForColumn(ColumnDefinition column) { - return indexesByColumn.get(column); + return indexesByColumn.get(column.name.bytes); } /** @@ -427,105 +401,125 @@ public class SecondaryIndexManager } /** - * When building an index against existing data, add the given row to the index - * - * @param key the row key - * @param cf the current rows data + * When building an index against existing data, add the given partition to the index */ - public void indexRow(ByteBuffer key, ColumnFamily cf, OpOrder.Group opGroup) + public void indexPartition(UnfilteredRowIterator partition, OpOrder.Group opGroup, Set allIndexes, int nowInSec) { - // Update entire row only once per row level index - Set> appliedRowLevelIndexes = null; + Set perRowIndexes = perRowIndexes(); + Set perColumnIndexes = perColumnIndexes(); - for (SecondaryIndex index : allIndexes) + if (!perRowIndexes.isEmpty()) { - if (index instanceof PerRowSecondaryIndex) - { - if (appliedRowLevelIndexes == null) - appliedRowLevelIndexes = new HashSet<>(); + // TODO: This is passing the same partition iterator to all perRow index, which means this only + // work if there is only one of them. We should change the API so it doesn't work directly on the + // partition, but rather on individual rows, so we can do a single iteration on the partition in this + // method and pass the rows to index to all indexes. + // Update entire partition only once per row level index + Set> appliedRowLevelIndexes = new HashSet<>(); + for (PerRowSecondaryIndex index : perRowIndexes) + { if (appliedRowLevelIndexes.add(index.getClass())) - ((PerRowSecondaryIndex)index).index(key, cf); - } - else - { - for (Cell cell : cf) - if (cell.isLive() && index.indexes(cell.name())) - ((PerColumnSecondaryIndex) index).insert(key, cell, opGroup); + ((PerRowSecondaryIndex)index).index(partition.partitionKey().getKey(), partition); } } - } - /** - * Delete all columns from all indexes for this row. For when cleanup rips a row out entirely. - * - * @param key the row key - * @param indexedColumnsInRow all column names in row - */ - public void deleteFromIndexes(DecoratedKey key, List indexedColumnsInRow, OpOrder.Group opGroup) - { - // Update entire row only once per row level index - Set> cleanedRowLevelIndexes = null; - - for (Cell cell : indexedColumnsInRow) + if (!perColumnIndexes.isEmpty()) { - for (SecondaryIndex index : indexFor(cell.name())) + DecoratedKey key = partition.partitionKey(); + + if (!partition.staticRow().isEmpty()) { - if (index instanceof PerRowSecondaryIndex) + for (PerColumnSecondaryIndex index : perColumnIndexes) + index.indexRow(key, partition.staticRow(), opGroup, nowInSec); + } + + try (RowIterator filtered = UnfilteredRowIterators.filter(partition, nowInSec)) + { + while (filtered.hasNext()) { - if (cleanedRowLevelIndexes == null) - cleanedRowLevelIndexes = new HashSet<>(); - if (cleanedRowLevelIndexes.add(index.getClass())) - ((PerRowSecondaryIndex) index).delete(key, opGroup); - } - else - { - ((PerColumnSecondaryIndex) index).deleteForCleanup(key.getKey(), cell, opGroup); + Row row = filtered.next(); + for (PerColumnSecondaryIndex index : perColumnIndexes) + index.indexRow(key, row, opGroup, nowInSec); } } } } /** - * This helper acts as a closure around the indexManager - * and updated cf data to ensure that down in - * Memtable's ColumnFamily implementation, the index - * can get updated. Note: only a CF backed by AtomicSortedColumns implements - * this behaviour fully, other types simply ignore the index updater. + * Delete all data from all indexes for this partition. For when cleanup rips a partition out entirely. */ - public Updater updaterFor(DecoratedKey key, ColumnFamily cf, OpOrder.Group opGroup) + public void deleteFromIndexes(UnfilteredRowIterator partition, OpOrder.Group opGroup, int nowInSec) + { + ByteBuffer key = partition.partitionKey().getKey(); + + for (PerRowSecondaryIndex index : perRowIndexes()) + index.delete(key, opGroup); + + Set indexes = perColumnIndexes(); + + while (partition.hasNext()) + { + Unfiltered unfiltered = partition.next(); + if (unfiltered.kind() != Unfiltered.Kind.ROW) + continue; + + Row row = (Row) unfiltered; + Clustering clustering = row.clustering(); + if (!row.deletion().isLive()) + for (PerColumnSecondaryIndex index : indexes) + index.maybeDelete(key, clustering, row.deletion(), opGroup); + for (Cell cell : row) + { + for (PerColumnSecondaryIndex index : indexes) + { + if (!index.indexes(cell.column())) + continue; + + ((PerColumnSecondaryIndex) index).deleteForCleanup(key, clustering, cell, opGroup, nowInSec); + } + } + } + } + + /** + * This helper acts as a closure around the indexManager and updated data + * to ensure that down in Memtable's ColumnFamily implementation, the index + * can get updated. + */ + public Updater updaterFor(PartitionUpdate update, OpOrder.Group opGroup, int nowInSec) { return (indexesByColumn.isEmpty() && rowLevelIndexMap.isEmpty()) ? nullUpdater - : new StandardUpdater(key, cf, opGroup); + : new StandardUpdater(update, opGroup, nowInSec); } /** * Updated closure with only the modified row key. */ - public Updater gcUpdaterFor(DecoratedKey key) + public Updater gcUpdaterFor(DecoratedKey key, int nowInSec) { - return new GCUpdater(key); + return new GCUpdater(key, nowInSec); } /** * Get a list of IndexSearchers from the union of expression index types - * @param clause the query clause + * @param command the query * @return the searchers needed to query the index */ - public List getIndexSearchersForQuery(List clause) + public List getIndexSearchersFor(ReadCommand command) { - Map> groupByIndexType = new HashMap<>(); + Map> groupByIndexType = new HashMap<>(); //Group columns by type - for (IndexExpression ix : clause) + for (RowFilter.Expression e : command.rowFilter()) { - SecondaryIndex index = getIndexForColumn(ix.column); + SecondaryIndex index = getIndexForColumn(e.column()); - if (index == null || !index.supportsOperator(ix.operator)) + if (index == null || !index.supportsOperator(e.operator())) continue; - Set columns = groupByIndexType.get(index.indexTypeForGrouping()); + Set columns = groupByIndexType.get(index.indexTypeForGrouping()); if (columns == null) { @@ -533,107 +527,54 @@ public class SecondaryIndexManager groupByIndexType.put(index.indexTypeForGrouping(), columns); } - columns.add(ix.column); + columns.add(e.column()); } List indexSearchers = new ArrayList<>(groupByIndexType.size()); //create searcher per type - for (Set column : groupByIndexType.values()) + for (Set column : groupByIndexType.values()) indexSearchers.add(getIndexForColumn(column.iterator().next()).createSecondaryIndexSearcher(column)); return indexSearchers; } - /** - * Validates an union of expression index types. It will throw a {@link RuntimeException} if - * any of the expressions in the provided clause is not valid for its index implementation. - * @param clause the query clause - * @throws org.apache.cassandra.exceptions.InvalidRequestException in case of validation errors - */ - public void validateIndexSearchersForQuery(List clause) throws InvalidRequestException + public SecondaryIndexSearcher getBestIndexSearcherFor(ReadCommand command) { - // Group by index type - Map> expressionsByIndexType = new HashMap<>(); - Map> columnsByIndexType = new HashMap<>(); - for (IndexExpression indexExpression : clause) + List indexSearchers = getIndexSearchersFor(command); + + if (indexSearchers.isEmpty()) + return null; + + SecondaryIndexSearcher mostSelective = null; + long bestEstimate = Long.MAX_VALUE; + for (SecondaryIndexSearcher searcher : indexSearchers) { - SecondaryIndex index = getIndexForColumn(indexExpression.column); - - if (index == null) - continue; - - String canonicalIndexName = index.getClass().getCanonicalName(); - Set expressions = expressionsByIndexType.get(canonicalIndexName); - Set columns = columnsByIndexType.get(canonicalIndexName); - if (expressions == null) + SecondaryIndex highestSelectivityIndex = searcher.highestSelectivityIndex(command.rowFilter()); + long estimate = highestSelectivityIndex.estimateResultRows(); + if (estimate <= bestEstimate) { - expressions = new HashSet<>(); - columns = new HashSet<>(); - expressionsByIndexType.put(canonicalIndexName, expressions); - columnsByIndexType.put(canonicalIndexName, columns); - } - - expressions.add(indexExpression); - columns.add(indexExpression.column); - } - - // Validate - boolean haveSupportedIndexLookup = false; - for (Map.Entry> expressions : expressionsByIndexType.entrySet()) - { - Set columns = columnsByIndexType.get(expressions.getKey()); - SecondaryIndex secondaryIndex = getIndexForColumn(columns.iterator().next()); - SecondaryIndexSearcher searcher = secondaryIndex.createSecondaryIndexSearcher(columns); - for (IndexExpression expression : expressions.getValue()) - { - searcher.validate(expression); - haveSupportedIndexLookup |= secondaryIndex.supportsOperator(expression.operator); + bestEstimate = estimate; + mostSelective = searcher; } } - - if (!haveSupportedIndexLookup) - { - // build the error message - int i = 0; - StringBuilder sb = new StringBuilder("No secondary indexes on the restricted columns support the provided operators: "); - for (Map.Entry> expressions : expressionsByIndexType.entrySet()) - { - for (IndexExpression expression : expressions.getValue()) - { - if (i++ > 0) - sb.append(", "); - sb.append("'"); - String columnName; - try - { - columnName = ByteBufferUtil.string(expression.column); - } - catch (CharacterCodingException ex) - { - columnName = ""; - } - sb.append(columnName).append(" ").append(expression.operator).append(" ").append("'"); - } - } - - throw new InvalidRequestException(sb.toString()); - } + return mostSelective; } /** - * Performs a search across a number of column indexes - * - * @param filter the column range to restrict to - * @return found indexed rows + * Validates an union of expression index types. It will throw an {@link InvalidRequestException} if + * any of the expressions in the provided clause is not valid for its index implementation. + * @param filter the filter to check + * @throws org.apache.cassandra.exceptions.InvalidRequestException in case of validation errors */ - public List search(ExtendedFilter filter) + public void validateFilter(RowFilter filter) throws InvalidRequestException { - SecondaryIndexSearcher mostSelective = getHighestSelectivityIndexSearcher(filter.getClause()); - if (mostSelective == null) - return Collections.emptyList(); - else - return mostSelective.search(filter); + for (RowFilter.Expression expression : filter) + { + SecondaryIndex index = getIndexForColumn(expression.column()); + if (index != null && index.supportsOperator(expression.operator())) + expression.validateForIndexing(); + } } public Set getIndexesByNames(Set idxNames) @@ -666,19 +607,27 @@ public class SecondaryIndexManager index.setIndexRemoved(); } - public SecondaryIndex validate(ByteBuffer rowKey, Cell cell) + public void validate(DecoratedKey partitionKey) throws InvalidRequestException { - for (SecondaryIndex index : indexFor(cell.name())) - { - if (!index.validate(rowKey, cell)) - return index; - } - return null; + for (SecondaryIndex index : perColumnIndexes()) + index.validate(partitionKey); + } + + public void validate(Clustering clustering) throws InvalidRequestException + { + for (SecondaryIndex index : perColumnIndexes()) + index.validate(clustering); + } + + public void validate(ColumnDefinition column, ByteBuffer value, CellPath path) throws InvalidRequestException + { + for (SecondaryIndex index : indexFor(column)) + index.validate(value, path); } static boolean shouldCleanupOldValue(Cell oldCell, Cell newCell) { - // If any one of name/value/timestamp are different, then we + // If either the value or timestamp is different, then we // should delete from the index. If not, then we can infer that // at least one of the cells is an ExpiringColumn and that the // difference is in the expiry time. In this case, we don't want to @@ -686,10 +635,9 @@ public class SecondaryIndexManager // will just hide the inserted value. // Completely identical cells (including expiring columns with // identical ttl & localExpirationTime) will not get this far due - // to the oldCell.equals(newColumn) in StandardUpdater.update - return !oldCell.name().equals(newCell.name()) - || !oldCell.value().equals(newCell.value()) - || oldCell.timestamp() != newCell.timestamp(); + // to the oldCell.equals(newCell) in StandardUpdater.update + return !oldCell.value().equals(newCell.value()) + || oldCell.livenessInfo().timestamp() != newCell.livenessInfo().timestamp(); } private Set filterByColumn(Set idxNames) @@ -712,14 +660,17 @@ public class SecondaryIndexManager public static interface Updater { + /** Called when a row with the provided clustering and row infos is inserted */ + public void maybeIndex(Clustering clustering, long timestamp, int ttl, DeletionTime deletion); + /** called when constructing the index against pre-existing data */ - public void insert(Cell cell); + public void insert(Clustering clustering, Cell cell); /** called when updating the index from a memtable */ - public void update(Cell oldCell, Cell cell); + public void update(Clustering clustering, Cell oldCell, Cell cell); /** called when lazy-updating the index during compaction (CASSANDRA-2897) */ - public void remove(Cell current); + public void remove(Clustering clustering, Cell current); /** called after memtable updates are complete (CASSANDRA-5397) */ public void updateRowLevelIndexes(); @@ -728,34 +679,38 @@ public class SecondaryIndexManager private final class GCUpdater implements Updater { private final DecoratedKey key; + private final int nowInSec; - public GCUpdater(DecoratedKey key) + public GCUpdater(DecoratedKey key, int nowInSec) { this.key = key; + this.nowInSec = nowInSec; } - public void insert(Cell cell) + public void maybeIndex(Clustering clustering, long timestamp, int ttl, DeletionTime deletion) { throw new UnsupportedOperationException(); } - public void update(Cell oldCell, Cell newCell) + public void insert(Clustering clustering, Cell cell) { throw new UnsupportedOperationException(); } - public void remove(Cell cell) + public void update(Clustering clustering, Cell oldCell, Cell newCell) { - if (!cell.isLive()) - return; + throw new UnsupportedOperationException(); + } - for (SecondaryIndex index : indexFor(cell.name())) + public void remove(Clustering clustering, Cell cell) + { + for (SecondaryIndex index : indexFor(cell.column())) { if (index instanceof PerColumnSecondaryIndex) { try (OpOrder.Group opGroup = baseCfs.keyspace.writeOrder.start()) { - ((PerColumnSecondaryIndex) index).delete(key.getKey(), cell, opGroup); + ((PerColumnSecondaryIndex) index).delete(key.getKey(), clustering, cell, opGroup, nowInSec); } } } @@ -770,39 +725,50 @@ public class SecondaryIndexManager private final class StandardUpdater implements Updater { - private final DecoratedKey key; - private final ColumnFamily cf; + private final PartitionUpdate update; private final OpOrder.Group opGroup; + private final int nowInSec; - public StandardUpdater(DecoratedKey key, ColumnFamily cf, OpOrder.Group opGroup) + public StandardUpdater(PartitionUpdate update, OpOrder.Group opGroup, int nowInSec) { - this.key = key; - this.cf = cf; + this.update = update; this.opGroup = opGroup; + this.nowInSec = nowInSec; } - public void insert(Cell cell) + public void maybeIndex(Clustering clustering, long timestamp, int ttl, DeletionTime deletion) { - if (!cell.isLive()) + for (PerColumnSecondaryIndex index : perColumnIndexes()) + { + if (timestamp != LivenessInfo.NO_TIMESTAMP) + index.maybeIndex(update.partitionKey().getKey(), clustering, timestamp, ttl, opGroup, nowInSec); + if (!deletion.isLive()) + index.maybeDelete(update.partitionKey().getKey(), clustering, deletion, opGroup); + } + } + + public void insert(Clustering clustering, Cell cell) + { + if (!cell.isLive(nowInSec)) return; - for (SecondaryIndex index : indexFor(cell.name())) + for (SecondaryIndex index : indexFor(cell.column())) if (index instanceof PerColumnSecondaryIndex) - ((PerColumnSecondaryIndex) index).insert(key.getKey(), cell, opGroup); + ((PerColumnSecondaryIndex) index).insert(update.partitionKey().getKey(), clustering, cell, opGroup); } - public void update(Cell oldCell, Cell cell) + public void update(Clustering clustering, Cell oldCell, Cell cell) { if (oldCell.equals(cell)) return; - for (SecondaryIndex index : indexFor(cell.name())) + for (SecondaryIndex index : indexFor(cell.column())) { if (index instanceof PerColumnSecondaryIndex) { - if (cell.isLive()) + if (cell.isLive(nowInSec)) { - ((PerColumnSecondaryIndex) index).update(key.getKey(), oldCell, cell, opGroup); + ((PerColumnSecondaryIndex) index).update(update.partitionKey().getKey(), clustering, oldCell, cell, opGroup, nowInSec); } else { @@ -812,53 +778,22 @@ public class SecondaryIndexManager // identical values and ttl) Then, we don't want to delete as the // tombstone will hide the new value we just inserted; see CASSANDRA-7268 if (shouldCleanupOldValue(oldCell, cell)) - ((PerColumnSecondaryIndex) index).delete(key.getKey(), oldCell, opGroup); + ((PerColumnSecondaryIndex) index).delete(update.partitionKey().getKey(), clustering, oldCell, opGroup, nowInSec); } } } } - public void remove(Cell cell) + public void remove(Clustering clustering, Cell cell) { - if (!cell.isLive()) - return; - - for (SecondaryIndex index : indexFor(cell.name())) - if (index instanceof PerColumnSecondaryIndex) - ((PerColumnSecondaryIndex) index).delete(key.getKey(), cell, opGroup); + throw new UnsupportedOperationException(); } public void updateRowLevelIndexes() { for (SecondaryIndex index : rowLevelIndexMap.values()) - ((PerRowSecondaryIndex) index).index(key.getKey(), cf); + ((PerRowSecondaryIndex) index).index(update.partitionKey().getKey(), update.unfilteredIterator()); } } - - public SecondaryIndexSearcher getHighestSelectivityIndexSearcher(List clause) - { - if (clause == null) - return null; - - List indexSearchers = getIndexSearchersForQuery(clause); - - if (indexSearchers.isEmpty()) - return null; - - SecondaryIndexSearcher mostSelective = null; - long bestEstimate = Long.MAX_VALUE; - for (SecondaryIndexSearcher searcher : indexSearchers) - { - SecondaryIndex highestSelectivityIndex = searcher.highestSelectivityIndex(clause); - long estimate = highestSelectivityIndex.estimateResultRows(); - if (estimate <= bestEstimate) - { - bestEstimate = estimate; - mostSelective = searcher; - } - } - - return mostSelective; - } } diff --git a/src/java/org/apache/cassandra/db/index/SecondaryIndexSearcher.java b/src/java/org/apache/cassandra/db/index/SecondaryIndexSearcher.java index 5812e9def4..4f63ae8604 100644 --- a/src/java/org/apache/cassandra/db/index/SecondaryIndexSearcher.java +++ b/src/java/org/apache/cassandra/db/index/SecondaryIndexSearcher.java @@ -20,75 +20,192 @@ package org.apache.cassandra.db.index; import java.nio.ByteBuffer; import java.util.*; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import org.apache.cassandra.config.CFMetaData; +import org.apache.cassandra.config.ColumnDefinition; import org.apache.cassandra.db.*; -import org.apache.cassandra.db.filter.ExtendedFilter; -import org.apache.cassandra.exceptions.InvalidRequestException; +import org.apache.cassandra.db.filter.*; +import org.apache.cassandra.db.partitions.PartitionIterator; +import org.apache.cassandra.db.partitions.UnfilteredPartitionIterator; +import org.apache.cassandra.db.rows.*; +import org.apache.cassandra.dht.AbstractBounds; import org.apache.cassandra.tracing.Tracing; import org.apache.cassandra.utils.FBUtilities; public abstract class SecondaryIndexSearcher { + private static final Logger logger = LoggerFactory.getLogger(SecondaryIndexSearcher.class); + protected final SecondaryIndexManager indexManager; - protected final Set columns; + protected final Set columns; protected final ColumnFamilyStore baseCfs; - public SecondaryIndexSearcher(SecondaryIndexManager indexManager, Set columns) + public SecondaryIndexSearcher(SecondaryIndexManager indexManager, Set columns) { this.indexManager = indexManager; this.columns = columns; this.baseCfs = indexManager.baseCfs; } - public SecondaryIndex highestSelectivityIndex(List clause) + public SecondaryIndex highestSelectivityIndex(RowFilter filter) { - IndexExpression expr = highestSelectivityPredicate(clause, false); - return expr == null ? null : indexManager.getIndexForColumn(expr.column); + RowFilter.Expression expr = highestSelectivityPredicate(filter, false); + return expr == null ? null : indexManager.getIndexForColumn(expr.column()); } - public abstract List search(ExtendedFilter filter); - - /** - * @return true this index is able to handle the given index expressions. - */ - public boolean canHandleIndexClause(List clause) + public RowFilter.Expression primaryClause(ReadCommand command) { - for (IndexExpression expression : clause) + return highestSelectivityPredicate(command.rowFilter(), false); + } + + @SuppressWarnings("resource") // Both the OpOrder and 'indexIter' are closed on exception, or through the closing of the result + // of this method. + public UnfilteredPartitionIterator search(ReadCommand command, ReadOrderGroup orderGroup) + { + RowFilter.Expression primary = highestSelectivityPredicate(command.rowFilter(), true); + assert primary != null; + + AbstractSimplePerColumnSecondaryIndex index = (AbstractSimplePerColumnSecondaryIndex)indexManager.getIndexForColumn(primary.column()); + assert index != null && index.getIndexCfs() != null; + + if (logger.isDebugEnabled()) + logger.debug("Most-selective indexed predicate is {}", primary); + + DecoratedKey indexKey = index.getIndexKeyFor(primary.getIndexValue()); + + UnfilteredRowIterator indexIter = queryIndex(index, indexKey, command, orderGroup); + try { - if (!columns.contains(expression.column)) - continue; - - SecondaryIndex index = indexManager.getIndexForColumn(expression.column); - if (index != null && index.getIndexCfs() != null && index.supportsOperator(expression.operator)) - return true; + return queryDataFromIndex(index, indexKey, UnfilteredRowIterators.filter(indexIter, command.nowInSec()), command, orderGroup); + } + catch (RuntimeException | Error e) + { + indexIter.close(); + throw e; } - return false; - } - - /** - * Validates the specified {@link IndexExpression}. It will throw an {@link org.apache.cassandra.exceptions.InvalidRequestException} - * if the provided clause is not valid for the index implementation. - * - * @param indexExpression An {@link IndexExpression} to be validated - * @throws org.apache.cassandra.exceptions.InvalidRequestException in case of validation errors - */ - public void validate(IndexExpression indexExpression) throws InvalidRequestException - { } - protected IndexExpression highestSelectivityPredicate(List clause, boolean includeInTrace) + private UnfilteredRowIterator queryIndex(AbstractSimplePerColumnSecondaryIndex index, DecoratedKey indexKey, ReadCommand command, ReadOrderGroup orderGroup) { - IndexExpression best = null; + ClusteringIndexFilter filter = makeIndexFilter(index, command); + CFMetaData indexMetadata = index.getIndexCfs().metadata; + return SinglePartitionReadCommand.create(indexMetadata, command.nowInSec(), indexKey, ColumnFilter.all(indexMetadata), filter) + .queryMemtableAndDisk(index.getIndexCfs(), orderGroup.indexReadOpOrderGroup()); + } + + private ClusteringIndexFilter makeIndexFilter(AbstractSimplePerColumnSecondaryIndex index, ReadCommand command) + { + if (command instanceof SinglePartitionReadCommand) + { + // Note: as yet there's no route to get here - a 2i query *always* uses a + // PartitionRangeReadCommand. This is here in preparation for coming changes + // in SelectStatement. + SinglePartitionReadCommand sprc = (SinglePartitionReadCommand)command; + ByteBuffer pk = sprc.partitionKey().getKey(); + ClusteringIndexFilter filter = sprc.clusteringIndexFilter(); + + if (filter instanceof ClusteringIndexNamesFilter) + { + NavigableSet requested = ((ClusteringIndexNamesFilter)filter).requestedRows(); + NavigableSet clusterings = new TreeSet<>(index.getIndexComparator()); + for (Clustering c : requested) + clusterings.add(index.makeIndexClustering(pk, c, (Cell)null).takeAlias()); + return new ClusteringIndexNamesFilter(clusterings, filter.isReversed()); + } + else + { + Slices requested = ((ClusteringIndexSliceFilter)filter).requestedSlices(); + Slices.Builder builder = new Slices.Builder(index.getIndexComparator()); + for (Slice slice : requested) + builder.add(index.makeIndexBound(pk, slice.start()), index.makeIndexBound(pk, slice.end())); + return new ClusteringIndexSliceFilter(builder.build(), filter.isReversed()); + } + } + else + { + + DataRange dataRange = ((PartitionRangeReadCommand)command).dataRange(); + AbstractBounds range = dataRange.keyRange(); + + Slice slice = Slice.ALL; + + /* + * XXX: If the range requested is a token range, we'll have to start at the beginning (and stop at the end) of + * the indexed row unfortunately (which will be inefficient), because we have no way to intuit the smallest possible + * key having a given token. A potential fix would be to actually store the token along the key in the indexed row. + */ + if (range.left instanceof DecoratedKey) + { + // the right hand side of the range may not be a DecoratedKey (for instance if we're paging), + // but if it is, we can optimise slightly by restricting the slice + if (range.right instanceof DecoratedKey) + { + + DecoratedKey startKey = (DecoratedKey) range.left; + DecoratedKey endKey = (DecoratedKey) range.right; + + Slice.Bound start = Slice.Bound.BOTTOM; + Slice.Bound end = Slice.Bound.TOP; + + /* + * For index queries over a range, we can't do a whole lot better than querying everything for the key range, though for + * slice queries where we can slightly restrict the beginning and end. + */ + if (!dataRange.isNamesQuery()) + { + ClusteringIndexSliceFilter startSliceFilter = ((ClusteringIndexSliceFilter) dataRange.clusteringIndexFilter(startKey)); + ClusteringIndexSliceFilter endSliceFilter = ((ClusteringIndexSliceFilter) dataRange.clusteringIndexFilter(endKey)); + + // We can't effectively support reversed queries when we have a range, so we don't support it + // (or through post-query reordering) and shouldn't get there. + assert !startSliceFilter.isReversed() && !endSliceFilter.isReversed(); + + Slices startSlices = startSliceFilter.requestedSlices(); + Slices endSlices = endSliceFilter.requestedSlices(); + + if (startSlices.size() > 0) + start = startSlices.get(0).start(); + + if (endSlices.size() > 0) + end = endSlices.get(endSlices.size() - 1).end(); + } + + slice = Slice.make(index.makeIndexBound(startKey.getKey(), start), + index.makeIndexBound(endKey.getKey(), end)); + } + else + { + // otherwise, just start the index slice from the key we do have + slice = Slice.make(index.makeIndexBound(((DecoratedKey)range.left).getKey(), Slice.Bound.BOTTOM), + Slice.Bound.TOP); + } + } + return new ClusteringIndexSliceFilter(Slices.with(index.getIndexComparator(), slice), false); + } + } + + protected abstract UnfilteredPartitionIterator queryDataFromIndex(AbstractSimplePerColumnSecondaryIndex index, + DecoratedKey indexKey, + RowIterator indexHits, + ReadCommand command, + ReadOrderGroup orderGroup); + + protected RowFilter.Expression highestSelectivityPredicate(RowFilter filter, boolean includeInTrace) + { + RowFilter.Expression best = null; int bestMeanCount = Integer.MAX_VALUE; Map candidates = new HashMap<>(); - for (IndexExpression expression : clause) + for (RowFilter.Expression expression : filter) { // skip columns belonging to a different index type - if (!columns.contains(expression.column)) + if (!columns.contains(expression.column())) continue; - SecondaryIndex index = indexManager.getIndexForColumn(expression.column); - if (index == null || index.getIndexCfs() == null || !index.supportsOperator(expression.operator)) + SecondaryIndex index = indexManager.getIndexForColumn(expression.column()); + if (index == null || index.getIndexCfs() == null || !index.supportsOperator(expression.operator())) continue; int columns = index.getIndexCfs().getMeanColumns(); @@ -106,34 +223,21 @@ public abstract class SecondaryIndexSearcher Tracing.trace("No applicable indexes found"); else if (Tracing.isTracing()) // pay for an additional threadlocal get() rather than build the strings unnecessarily - Tracing.trace("Candidate index mean cardinalities are {}. Scanning with {}.", - FBUtilities.toString(candidates), - indexManager.getIndexForColumn(best.column).getIndexName()); + Tracing.trace("Candidate index mean cardinalities are {}. Scanning with {}.", FBUtilities.toString(candidates), indexManager.getIndexForColumn(best.column()).getIndexName()); } return best; } /** - * Returns {@code true} if the specified list of {@link IndexExpression}s require a full scan of all the nodes. - * - * @param clause A list of {@link IndexExpression}s - * @return {@code true} if the {@code IndexExpression}s require a full scan, {@code false} otherwise - */ - public boolean requiresScanningAllRanges(List clause) - { - return false; - } - - /** - * Combines index query results from multiple nodes. This is done by the coordinator node after it has reconciled + * Post-process the result of an index query. This is done by the coordinator node after it has reconciled * the replica responses. * - * @param clause A list of {@link IndexExpression}s - * @param rows The index query results to be combined - * @return The combination of the index query results + * @param command The {@code ReadCommand} use for the query. + * @param result The index query results to be post-processed + * @return The post-processed results */ - public List postReconciliationProcessing(List clause, List rows) + public PartitionIterator postReconciliationProcessing(RowFilter filter, PartitionIterator result) { - return rows; + return result; } } diff --git a/src/java/org/apache/cassandra/db/index/composites/CompositesIndex.java b/src/java/org/apache/cassandra/db/index/composites/CompositesIndex.java index e88d456560..9333bcfc61 100644 --- a/src/java/org/apache/cassandra/db/index/composites/CompositesIndex.java +++ b/src/java/org/apache/cassandra/db/index/composites/CompositesIndex.java @@ -26,36 +26,20 @@ import org.apache.cassandra.utils.concurrent.OpOrder; import org.apache.cassandra.config.CFMetaData; import org.apache.cassandra.config.ColumnDefinition; import org.apache.cassandra.db.*; -import org.apache.cassandra.db.composites.CellName; -import org.apache.cassandra.db.composites.CellNameType; -import org.apache.cassandra.db.composites.Composite; +import org.apache.cassandra.db.rows.*; +import org.apache.cassandra.db.partitions.*; import org.apache.cassandra.db.index.AbstractSimplePerColumnSecondaryIndex; import org.apache.cassandra.db.index.SecondaryIndex; import org.apache.cassandra.db.index.SecondaryIndexManager; import org.apache.cassandra.db.index.SecondaryIndexSearcher; -import org.apache.cassandra.db.marshal.AbstractType; import org.apache.cassandra.db.marshal.CollectionType; import org.apache.cassandra.exceptions.ConfigurationException; /** - * Base class for secondary indexes where composites are involved. + * Base class for internal secondary indexes (this could be merged with AbstractSimplePerColumnSecondaryIndex). */ public abstract class CompositesIndex extends AbstractSimplePerColumnSecondaryIndex { - private volatile CellNameType indexComparator; - - protected CellNameType getIndexComparator() - { - // Yes, this is racy, but doing this more than once is not a big deal, we just want to avoid doing it every time - // More seriously, we should fix that whole SecondaryIndex API so this can be a final and avoid all that non-sense. - if (indexComparator == null) - { - assert columnDef != null; - indexComparator = getIndexComparator(baseCfs.metadata, columnDef); - } - return indexComparator; - } - public static CompositesIndex create(ColumnDefinition cfDef) { if (cfDef.type.isCollection() && cfDef.type.isMultiCell()) @@ -90,68 +74,56 @@ public abstract class CompositesIndex extends AbstractSimplePerColumnSecondaryIn throw new AssertionError(); } - // Check SecondaryIndex.getIndexComparator if you want to know why this is static - public static CellNameType getIndexComparator(CFMetaData baseMetadata, ColumnDefinition cfDef) + public static void addIndexClusteringColumns(CFMetaData.Builder indexMetadata, CFMetaData baseMetadata, ColumnDefinition cfDef) { if (cfDef.type.isCollection() && cfDef.type.isMultiCell()) { - switch (((CollectionType)cfDef.type).kind) + CollectionType type = (CollectionType)cfDef.type; + if (type.kind == CollectionType.Kind.LIST + || (type.kind == CollectionType.Kind.MAP && cfDef.hasIndexOption(SecondaryIndex.INDEX_VALUES_OPTION_NAME))) { - case LIST: - return CompositesIndexOnCollectionValue.buildIndexComparator(baseMetadata, cfDef); - case SET: - return CompositesIndexOnCollectionKey.buildIndexComparator(baseMetadata, cfDef); - case MAP: - if (cfDef.hasIndexOption(SecondaryIndex.INDEX_KEYS_OPTION_NAME)) - return CompositesIndexOnCollectionKey.buildIndexComparator(baseMetadata, cfDef); - else if (cfDef.hasIndexOption(SecondaryIndex.INDEX_ENTRIES_OPTION_NAME)) - return CompositesIndexOnCollectionKeyAndValue.buildIndexComparator(baseMetadata, cfDef); - else - return CompositesIndexOnCollectionValue.buildIndexComparator(baseMetadata, cfDef); + CompositesIndexOnCollectionValue.addClusteringColumns(indexMetadata, baseMetadata, cfDef); + } + else + { + addGenericClusteringColumns(indexMetadata, baseMetadata, cfDef); } } - - switch (cfDef.kind) + else if (cfDef.isClusteringColumn()) { - case CLUSTERING_COLUMN: - return CompositesIndexOnClusteringKey.buildIndexComparator(baseMetadata, cfDef); - case REGULAR: - return CompositesIndexOnRegular.buildIndexComparator(baseMetadata, cfDef); - case PARTITION_KEY: - return CompositesIndexOnPartitionKey.buildIndexComparator(baseMetadata, cfDef); - //case COMPACT_VALUE: - // return CompositesIndexOnCompactValue.buildIndexComparator(baseMetadata, cfDef); + CompositesIndexOnClusteringKey.addClusteringColumns(indexMetadata, baseMetadata, cfDef); + } + else + { + addGenericClusteringColumns(indexMetadata, baseMetadata, cfDef); } - throw new AssertionError(); } - protected CellName makeIndexColumnName(ByteBuffer rowKey, Cell cell) + protected static void addGenericClusteringColumns(CFMetaData.Builder indexMetadata, CFMetaData baseMetadata, ColumnDefinition columnDef) { - return getIndexComparator().create(makeIndexColumnPrefix(rowKey, cell.name()), null); + indexMetadata.addClusteringColumn("partition_key", SecondaryIndex.keyComparator); + for (ColumnDefinition def : baseMetadata.clusteringColumns()) + indexMetadata.addClusteringColumn(def.name, def.type); } - protected abstract Composite makeIndexColumnPrefix(ByteBuffer rowKey, Composite columnName); + public abstract IndexedEntry decodeEntry(DecoratedKey indexedValue, Row indexEntry); - public abstract IndexedEntry decodeEntry(DecoratedKey indexedValue, Cell indexEntry); + public abstract boolean isStale(Row row, ByteBuffer indexValue, int nowInSec); - public abstract boolean isStale(IndexedEntry entry, ColumnFamily data, long now); - - public void delete(IndexedEntry entry, OpOrder.Group opGroup) + public void delete(IndexedEntry entry, OpOrder.Group opGroup, int nowInSec) { - int localDeletionTime = (int) (System.currentTimeMillis() / 1000); - ColumnFamily cfi = ArrayBackedSortedColumns.factory.create(indexCfs.metadata); - cfi.addTombstone(entry.indexEntry, localDeletionTime, entry.timestamp); - indexCfs.apply(entry.indexValue, cfi, SecondaryIndexManager.nullUpdater, opGroup, null); + PartitionUpdate upd = new PartitionUpdate(indexCfs.metadata, entry.indexValue, PartitionColumns.NONE, 1); + Row.Writer writer = upd.writer(); + Rows.writeClustering(entry.indexClustering, writer); + writer.writeRowDeletion(new SimpleDeletionTime(entry.timestamp, nowInSec)); + writer.endOfRow(); + indexCfs.apply(upd, SecondaryIndexManager.nullUpdater, opGroup, null); + if (logger.isDebugEnabled()) - logger.debug("removed index entry for cleaned-up value {}:{}", entry.indexValue, cfi); + logger.debug("removed index entry for cleaned-up value {}:{}", entry.indexValue, upd); } - protected AbstractType getExpressionComparator() - { - return baseCfs.metadata.getColumnDefinitionComparator(columnDef); - } - - public SecondaryIndexSearcher createSecondaryIndexSearcher(Set columns) + public SecondaryIndexSearcher createSecondaryIndexSearcher(Set columns) { return new CompositesSearcher(baseCfs.indexManager, columns); } @@ -178,31 +150,19 @@ public abstract class CompositesIndex extends AbstractSimplePerColumnSecondaryIn public static class IndexedEntry { public final DecoratedKey indexValue; - public final CellName indexEntry; + public final Clustering indexClustering; public final long timestamp; public final ByteBuffer indexedKey; - public final Composite indexedEntryPrefix; - public final ByteBuffer indexedEntryCollectionKey; // may be null + public final Clustering indexedEntryClustering; - public IndexedEntry(DecoratedKey indexValue, CellName indexEntry, long timestamp, ByteBuffer indexedKey, Composite indexedEntryPrefix) - { - this(indexValue, indexEntry, timestamp, indexedKey, indexedEntryPrefix, null); - } - - public IndexedEntry(DecoratedKey indexValue, - CellName indexEntry, - long timestamp, - ByteBuffer indexedKey, - Composite indexedEntryPrefix, - ByteBuffer indexedEntryCollectionKey) + public IndexedEntry(DecoratedKey indexValue, Clustering indexClustering, long timestamp, ByteBuffer indexedKey, Clustering indexedEntryClustering) { this.indexValue = indexValue; - this.indexEntry = indexEntry; + this.indexClustering = indexClustering.takeAlias(); this.timestamp = timestamp; this.indexedKey = indexedKey; - this.indexedEntryPrefix = indexedEntryPrefix; - this.indexedEntryCollectionKey = indexedEntryCollectionKey; + this.indexedEntryClustering = indexedEntryClustering.takeAlias(); } } } diff --git a/src/java/org/apache/cassandra/db/index/composites/CompositesIndexIncludingCollectionKey.java b/src/java/org/apache/cassandra/db/index/composites/CompositesIndexIncludingCollectionKey.java index 402ea05757..7624c1f74b 100644 --- a/src/java/org/apache/cassandra/db/index/composites/CompositesIndexIncludingCollectionKey.java +++ b/src/java/org/apache/cassandra/db/index/composites/CompositesIndexIncludingCollectionKey.java @@ -18,19 +18,9 @@ package org.apache.cassandra.db.index.composites; import java.nio.ByteBuffer; -import java.util.ArrayList; -import java.util.List; -import org.apache.cassandra.config.CFMetaData; -import org.apache.cassandra.config.ColumnDefinition; import org.apache.cassandra.db.*; -import org.apache.cassandra.db.composites.CBuilder; -import org.apache.cassandra.db.composites.CellName; -import org.apache.cassandra.db.composites.CellNameType; -import org.apache.cassandra.db.composites.Composite; -import org.apache.cassandra.db.composites.CompoundDenseCellNameType; -import org.apache.cassandra.db.index.SecondaryIndex; -import org.apache.cassandra.db.marshal.*; +import org.apache.cassandra.db.rows.*; /** * Common superclass for indexes that capture collection keys, including @@ -49,41 +39,22 @@ import org.apache.cassandra.db.marshal.*; */ public abstract class CompositesIndexIncludingCollectionKey extends CompositesIndex { - public static CellNameType buildIndexComparator(CFMetaData baseMetadata, ColumnDefinition columnDef) + protected CBuilder buildIndexClusteringPrefix(ByteBuffer rowKey, ClusteringPrefix prefix, CellPath path) { - int count = 1 + baseMetadata.clusteringColumns().size(); // row key + clustering prefix - List> types = new ArrayList>(count); - types.add(SecondaryIndex.keyComparator); - for (int i = 0; i < count - 1; i++) - types.add(baseMetadata.comparator.subtype(i)); - return new CompoundDenseCellNameType(types); - } - - protected Composite makeIndexColumnPrefix(ByteBuffer rowKey, Composite cellName) - { - int count = 1 + baseCfs.metadata.clusteringColumns().size(); - CBuilder builder = getIndexComparator().builder(); + CBuilder builder = CBuilder.create(getIndexComparator()); builder.add(rowKey); - for (int i = 0; i < Math.min(cellName.size(), count - 1); i++) - builder.add(cellName.get(i)); - return builder.build(); + for (int i = 0; i < prefix.size(); i++) + builder.add(prefix.get(i)); + return builder; } - public IndexedEntry decodeEntry(DecoratedKey indexedValue, Cell indexEntry) + public IndexedEntry decodeEntry(DecoratedKey indexedValue, Row indexEntry) { int count = 1 + baseCfs.metadata.clusteringColumns().size(); - CBuilder builder = baseCfs.getComparator().builder(); + Clustering clustering = indexEntry.clustering(); + CBuilder builder = CBuilder.create(baseCfs.getComparator()); for (int i = 0; i < count - 1; i++) - builder.add(indexEntry.name().get(i + 1)); - return new IndexedEntry(indexedValue, indexEntry.name(), indexEntry.timestamp(), indexEntry.name().get(0), builder.build()); - } - - @Override - public boolean indexes(CellName name) - { - // We index if the CQL3 column name is the one of the collection we index - AbstractType comp = baseCfs.metadata.getColumnDefinitionComparator(columnDef); - return name.size() > columnDef.position() - && comp.compare(name.get(columnDef.position()), columnDef.name.bytes) == 0; + builder.add(clustering.get(i + 1)); + return new IndexedEntry(indexedValue, clustering, indexEntry.primaryKeyLivenessInfo().timestamp(), clustering.get(0), builder.build()); } } diff --git a/src/java/org/apache/cassandra/db/index/composites/CompositesIndexOnClusteringKey.java b/src/java/org/apache/cassandra/db/index/composites/CompositesIndexOnClusteringKey.java index 0243b0d0f1..add444547b 100644 --- a/src/java/org/apache/cassandra/db/index/composites/CompositesIndexOnClusteringKey.java +++ b/src/java/org/apache/cassandra/db/index/composites/CompositesIndexOnClusteringKey.java @@ -18,15 +18,13 @@ package org.apache.cassandra.db.index.composites; import java.nio.ByteBuffer; -import java.util.ArrayList; import java.util.List; import org.apache.cassandra.config.CFMetaData; import org.apache.cassandra.config.ColumnDefinition; import org.apache.cassandra.db.*; -import org.apache.cassandra.db.composites.*; +import org.apache.cassandra.db.rows.*; import org.apache.cassandra.db.index.SecondaryIndex; -import org.apache.cassandra.db.marshal.*; import org.apache.cassandra.utils.concurrent.OpOrder; /** @@ -48,67 +46,90 @@ import org.apache.cassandra.utils.concurrent.OpOrder; */ public class CompositesIndexOnClusteringKey extends CompositesIndex { - public static CellNameType buildIndexComparator(CFMetaData baseMetadata, ColumnDefinition columnDef) + public static void addClusteringColumns(CFMetaData.Builder indexMetadata, CFMetaData baseMetadata, ColumnDefinition columnDef) { - // Index cell names are rk ck_0 ... ck_{i-1} ck_{i+1} ck_n, so n - // components total (where n is the number of clustering keys) - int ckCount = baseMetadata.clusteringColumns().size(); - List> types = new ArrayList>(ckCount); - types.add(SecondaryIndex.keyComparator); + indexMetadata.addClusteringColumn("partition_key", SecondaryIndex.keyComparator); + + List cks = baseMetadata.clusteringColumns(); for (int i = 0; i < columnDef.position(); i++) - types.add(baseMetadata.clusteringColumns().get(i).type); - for (int i = columnDef.position() + 1; i < ckCount; i++) - types.add(baseMetadata.clusteringColumns().get(i).type); - return new CompoundDenseCellNameType(types); + { + ColumnDefinition def = cks.get(i); + indexMetadata.addClusteringColumn(def.name, def.type); + } + for (int i = columnDef.position() + 1; i < cks.size(); i++) + { + ColumnDefinition def = cks.get(i); + indexMetadata.addClusteringColumn(def.name, def.type); + } } - protected ByteBuffer getIndexedValue(ByteBuffer rowKey, Cell cell) + protected ByteBuffer getIndexedValue(ByteBuffer rowKey, Clustering clustering, ByteBuffer cellValue, CellPath path) { - return cell.name().get(columnDef.position()); + return clustering.get(columnDef.position()); } - protected Composite makeIndexColumnPrefix(ByteBuffer rowKey, Composite columnName) + protected CBuilder buildIndexClusteringPrefix(ByteBuffer rowKey, ClusteringPrefix prefix, CellPath path) { - int count = Math.min(baseCfs.metadata.clusteringColumns().size(), columnName.size()); - CBuilder builder = getIndexComparator().prefixBuilder(); + CBuilder builder = CBuilder.create(getIndexComparator()); builder.add(rowKey); - for (int i = 0; i < Math.min(columnDef.position(), count); i++) - builder.add(columnName.get(i)); - for (int i = columnDef.position() + 1; i < count; i++) - builder.add(columnName.get(i)); - return builder.build(); + for (int i = 0; i < Math.min(columnDef.position(), prefix.size()); i++) + builder.add(prefix.get(i)); + for (int i = columnDef.position() + 1; i < prefix.size(); i++) + builder.add(prefix.get(i)); + return builder; } - public IndexedEntry decodeEntry(DecoratedKey indexedValue, Cell indexEntry) + public IndexedEntry decodeEntry(DecoratedKey indexedValue, Row indexEntry) { int ckCount = baseCfs.metadata.clusteringColumns().size(); - CBuilder builder = baseCfs.getComparator().builder(); + Clustering clustering = indexEntry.clustering(); + CBuilder builder = CBuilder.create(baseCfs.getComparator()); for (int i = 0; i < columnDef.position(); i++) - builder.add(indexEntry.name().get(i + 1)); + builder.add(clustering.get(i + 1)); builder.add(indexedValue.getKey()); for (int i = columnDef.position() + 1; i < ckCount; i++) - builder.add(indexEntry.name().get(i)); + builder.add(clustering.get(i)); - return new IndexedEntry(indexedValue, indexEntry.name(), indexEntry.timestamp(), indexEntry.name().get(0), builder.build()); + return new IndexedEntry(indexedValue, clustering, indexEntry.primaryKeyLivenessInfo().timestamp(), clustering.get(0), builder.build()); } @Override - public boolean indexes(CellName name) + protected boolean indexPrimaryKeyColumn() { - // For now, assume this is only used in CQL3 when we know name has enough component. return true; } - public boolean isStale(IndexedEntry entry, ColumnFamily data, long now) + @Override + public boolean indexes(ColumnDefinition c) { - return data.hasOnlyTombstones(now); + // Actual indexing for this index type is done through maybeIndex + return false; + } + + public boolean isStale(Row data, ByteBuffer indexValue, int nowInSec) + { + return !data.hasLiveData(nowInSec); } @Override - public void delete(ByteBuffer rowKey, Cell cell, OpOrder.Group opGroup) + public void maybeIndex(ByteBuffer partitionKey, Clustering clustering, long timestamp, int ttl, OpOrder.Group opGroup, int nowInSec) + { + if (clustering != Clustering.STATIC_CLUSTERING && clustering.get(columnDef.position()) != null) + insert(partitionKey, clustering, null, SimpleLivenessInfo.forUpdate(timestamp, ttl, nowInSec, indexCfs.metadata), opGroup); + } + + @Override + public void maybeDelete(ByteBuffer partitionKey, Clustering clustering, DeletionTime deletion, OpOrder.Group opGroup) + { + if (clustering.get(columnDef.position()) != null && !deletion.isLive()) + delete(partitionKey, clustering, null, null, deletion, opGroup); + } + + @Override + public void delete(ByteBuffer rowKey, Clustering clustering, Cell cell, OpOrder.Group opGroup, int nowInSec) { // We only know that one column of the CQL row has been updated/deleted, but we don't know if the // full row has been deleted so we should not do anything. If it ends up that the whole row has diff --git a/src/java/org/apache/cassandra/db/index/composites/CompositesIndexOnCollectionKey.java b/src/java/org/apache/cassandra/db/index/composites/CompositesIndexOnCollectionKey.java index 1e40710b68..50e81c426a 100644 --- a/src/java/org/apache/cassandra/db/index/composites/CompositesIndexOnCollectionKey.java +++ b/src/java/org/apache/cassandra/db/index/composites/CompositesIndexOnCollectionKey.java @@ -21,7 +21,7 @@ import java.nio.ByteBuffer; import org.apache.cassandra.cql3.Operator; import org.apache.cassandra.db.*; -import org.apache.cassandra.db.composites.CellName; +import org.apache.cassandra.db.rows.*; import org.apache.cassandra.db.marshal.*; /** @@ -38,22 +38,21 @@ public class CompositesIndexOnCollectionKey extends CompositesIndexIncludingColl return ((CollectionType)columnDef.type).nameComparator(); } - protected ByteBuffer getIndexedValue(ByteBuffer rowKey, Cell cell) + protected ByteBuffer getIndexedValue(ByteBuffer rowKey, Clustering clustering, ByteBuffer cellValue, CellPath path) { - return cell.name().get(columnDef.position() + 1); + return path.get(0); } @Override public boolean supportsOperator(Operator operator) { return operator == Operator.CONTAINS_KEY || - operator == Operator.CONTAINS && columnDef.type instanceof SetType; + operator == Operator.CONTAINS && columnDef.type instanceof SetType; } - public boolean isStale(IndexedEntry entry, ColumnFamily data, long now) + public boolean isStale(Row data, ByteBuffer indexValue, int nowInSec) { - CellName name = data.getComparator().create(entry.indexedEntryPrefix, columnDef, entry.indexValue.getKey()); - Cell cell = data.getColumn(name); - return cell == null || !cell.isLive(now); + Cell cell = data.getCell(columnDef, CellPath.create(indexValue)); + return cell == null || !cell.isLive(nowInSec); } } diff --git a/src/java/org/apache/cassandra/db/index/composites/CompositesIndexOnCollectionKeyAndValue.java b/src/java/org/apache/cassandra/db/index/composites/CompositesIndexOnCollectionKeyAndValue.java index 0b7f579ad8..766f8039ff 100644 --- a/src/java/org/apache/cassandra/db/index/composites/CompositesIndexOnCollectionKeyAndValue.java +++ b/src/java/org/apache/cassandra/db/index/composites/CompositesIndexOnCollectionKeyAndValue.java @@ -20,7 +20,7 @@ package org.apache.cassandra.db.index.composites; import java.nio.ByteBuffer; import org.apache.cassandra.db.*; -import org.apache.cassandra.db.composites.CellName; +import org.apache.cassandra.db.rows.*; import org.apache.cassandra.db.marshal.*; /** @@ -38,50 +38,22 @@ public class CompositesIndexOnCollectionKeyAndValue extends CompositesIndexInclu return CompositeType.getInstance(colType.nameComparator(), colType.valueComparator()); } - @Override - protected ByteBuffer getIndexedValue(ByteBuffer rowKey, Cell cell) + protected ByteBuffer getIndexedValue(ByteBuffer rowKey, Clustering clustering, ByteBuffer cellValue, CellPath path) { - final ByteBuffer key = cell.name().get(columnDef.position() + 1); - final ByteBuffer value = cell.value(); - return CompositeType.build(key, value); + return CompositeType.build(path.get(0), cellValue); } - @Override - public boolean isStale(IndexedEntry entry, ColumnFamily data, long now) + public boolean isStale(Row data, ByteBuffer indexValue, int nowInSec) { - Cell cell = extractTargetCell(entry, data); - if (cellIsDead(cell, now)) + ByteBuffer[] components = ((CompositeType)getIndexKeyComparator()).split(indexValue); + ByteBuffer mapKey = components[0]; + ByteBuffer mapValue = components[1]; + + Cell cell = data.getCell(columnDef, CellPath.create(mapKey)); + if (cell == null || !cell.isLive(nowInSec)) return true; - ByteBuffer indexCollectionValue = extractCollectionValue(entry); - ByteBuffer targetCollectionValue = cell.value(); + AbstractType valueComparator = ((CollectionType)columnDef.type).valueComparator(); - return valueComparator.compare(indexCollectionValue, targetCollectionValue) != 0; - } - - private Cell extractTargetCell(IndexedEntry entry, ColumnFamily data) - { - ByteBuffer collectionKey = extractCollectionKey(entry); - CellName name = data.getComparator().create(entry.indexedEntryPrefix, columnDef, collectionKey); - return data.getColumn(name); - } - - private ByteBuffer extractCollectionKey(IndexedEntry entry) - { - return extractIndexKeyComponent(entry, 0); - } - - private ByteBuffer extractIndexKeyComponent(IndexedEntry entry, int component) - { - return CompositeType.extractComponent(entry.indexValue.getKey(), component); - } - - private ByteBuffer extractCollectionValue(IndexedEntry entry) - { - return extractIndexKeyComponent(entry, 1); - } - - private boolean cellIsDead(Cell cell, long now) - { - return cell == null || !cell.isLive(now); + return valueComparator.compare(mapValue, cell.value()) != 0; } } diff --git a/src/java/org/apache/cassandra/db/index/composites/CompositesIndexOnCollectionValue.java b/src/java/org/apache/cassandra/db/index/composites/CompositesIndexOnCollectionValue.java index a11a0d95f9..5af842cbab 100644 --- a/src/java/org/apache/cassandra/db/index/composites/CompositesIndexOnCollectionValue.java +++ b/src/java/org/apache/cassandra/db/index/composites/CompositesIndexOnCollectionValue.java @@ -18,19 +18,13 @@ package org.apache.cassandra.db.index.composites; import java.nio.ByteBuffer; -import java.util.ArrayList; -import java.util.List; +import java.util.Iterator; import org.apache.cassandra.config.CFMetaData; import org.apache.cassandra.config.ColumnDefinition; import org.apache.cassandra.cql3.Operator; import org.apache.cassandra.db.*; -import org.apache.cassandra.db.composites.CBuilder; -import org.apache.cassandra.db.composites.CellName; -import org.apache.cassandra.db.composites.CellNameType; -import org.apache.cassandra.db.composites.Composite; -import org.apache.cassandra.db.composites.CompoundDenseCellNameType; -import org.apache.cassandra.db.index.SecondaryIndex; +import org.apache.cassandra.db.rows.*; import org.apache.cassandra.db.marshal.*; /** @@ -46,15 +40,12 @@ import org.apache.cassandra.db.marshal.*; */ public class CompositesIndexOnCollectionValue extends CompositesIndex { - public static CellNameType buildIndexComparator(CFMetaData baseMetadata, ColumnDefinition columnDef) + public static void addClusteringColumns(CFMetaData.Builder indexMetadata, CFMetaData baseMetadata, ColumnDefinition columnDef) { - int prefixSize = columnDef.position(); - List> types = new ArrayList<>(prefixSize + 2); - types.add(SecondaryIndex.keyComparator); - for (int i = 0; i < prefixSize; i++) - types.add(baseMetadata.comparator.subtype(i)); - types.add(((CollectionType)columnDef.type).nameComparator()); // collection key - return new CompoundDenseCellNameType(types); + addGenericClusteringColumns(indexMetadata, baseMetadata, columnDef); + + // collection key + indexMetadata.addClusteringColumn("cell_path", ((CollectionType)columnDef.type).nameComparator()); } @Override @@ -63,36 +54,32 @@ public class CompositesIndexOnCollectionValue extends CompositesIndex return ((CollectionType)columnDef.type).valueComparator(); } - protected ByteBuffer getIndexedValue(ByteBuffer rowKey, Cell cell) + protected ByteBuffer getIndexedValue(ByteBuffer rowKey, Clustering clustering, ByteBuffer cellValue, CellPath path) { - return cell.value(); + return cellValue; } - protected Composite makeIndexColumnPrefix(ByteBuffer rowKey, Composite cellName) + protected CBuilder buildIndexClusteringPrefix(ByteBuffer rowKey, ClusteringPrefix prefix, CellPath path) { - CBuilder builder = getIndexComparator().prefixBuilder(); + CBuilder builder = CBuilder.create(getIndexComparator()); builder.add(rowKey); - for (int i = 0; i < Math.min(columnDef.position(), cellName.size()); i++) - builder.add(cellName.get(i)); + for (int i = 0; i < prefix.size(); i++) + builder.add(prefix.get(i)); - // When indexing, cellName is a full name including the collection - // key. When searching, restricted clustering columns are included - // but the collection key is not. In this case, don't try to add an - // element to the builder for it, as it will just end up null and - // error out when retrieving cells from the index cf (CASSANDRA-7525) - if (cellName.size() >= columnDef.position() + 1) - builder.add(cellName.get(columnDef.position() + 1)); - return builder.build(); + // When indexing, cell will be present, but when searching, it won't (CASSANDRA-7525) + if (prefix.size() == baseCfs.metadata.clusteringColumns().size() && path != null) + builder.add(path.get(0)); + + return builder; } - public IndexedEntry decodeEntry(DecoratedKey indexedValue, Cell indexEntry) + public IndexedEntry decodeEntry(DecoratedKey indexedValue, Row indexEntry) { - int prefixSize = columnDef.position(); - CellName name = indexEntry.name(); - CBuilder builder = baseCfs.getComparator().builder(); - for (int i = 0; i < prefixSize; i++) - builder.add(name.get(i + 1)); - return new IndexedEntry(indexedValue, name, indexEntry.timestamp(), name.get(0), builder.build(), name.get(prefixSize + 1)); + Clustering clustering = indexEntry.clustering(); + CBuilder builder = CBuilder.create(baseCfs.getComparator()); + for (int i = 0; i < baseCfs.getComparator().size(); i++) + builder.add(clustering.get(i + 1)); + return new IndexedEntry(indexedValue, clustering, indexEntry.primaryKeyLivenessInfo().timestamp(), clustering.get(0), builder.build()); } @Override @@ -101,18 +88,15 @@ public class CompositesIndexOnCollectionValue extends CompositesIndex return operator == Operator.CONTAINS && !(columnDef.type instanceof SetType); } - @Override - public boolean indexes(CellName name) + public boolean isStale(Row data, ByteBuffer indexValue, int nowInSec) { - AbstractType comp = baseCfs.metadata.getColumnDefinitionComparator(columnDef); - return name.size() > columnDef.position() - && comp.compare(name.get(columnDef.position()), columnDef.name.bytes) == 0; - } - - public boolean isStale(IndexedEntry entry, ColumnFamily data, long now) - { - CellName name = data.getComparator().create(entry.indexedEntryPrefix, columnDef, entry.indexedEntryCollectionKey); - Cell cell = data.getColumn(name); - return cell == null || !cell.isLive(now) || ((CollectionType) columnDef.type).valueComparator().compare(entry.indexValue.getKey(), cell.value()) != 0; + Iterator iter = data.getCells(columnDef); + while (iter.hasNext()) + { + Cell cell = iter.next(); + if (cell.isLive(nowInSec) && ((CollectionType) columnDef.type).valueComparator().compare(indexValue, cell.value()) == 0) + return false; + } + return true; } } diff --git a/src/java/org/apache/cassandra/db/index/composites/CompositesIndexOnPartitionKey.java b/src/java/org/apache/cassandra/db/index/composites/CompositesIndexOnPartitionKey.java index df43057925..d48e58bcf8 100644 --- a/src/java/org/apache/cassandra/db/index/composites/CompositesIndexOnPartitionKey.java +++ b/src/java/org/apache/cassandra/db/index/composites/CompositesIndexOnPartitionKey.java @@ -18,14 +18,10 @@ package org.apache.cassandra.db.index.composites; import java.nio.ByteBuffer; -import java.util.ArrayList; -import java.util.List; -import org.apache.cassandra.config.CFMetaData; import org.apache.cassandra.config.ColumnDefinition; import org.apache.cassandra.db.*; -import org.apache.cassandra.db.composites.*; -import org.apache.cassandra.db.index.SecondaryIndex; +import org.apache.cassandra.db.rows.*; import org.apache.cassandra.db.marshal.*; import org.apache.cassandra.utils.concurrent.OpOrder; @@ -49,57 +45,59 @@ import org.apache.cassandra.utils.concurrent.OpOrder; */ public class CompositesIndexOnPartitionKey extends CompositesIndex { - public static CellNameType buildIndexComparator(CFMetaData baseMetadata, ColumnDefinition columnDef) - { - int ckCount = baseMetadata.clusteringColumns().size(); - List> types = new ArrayList>(ckCount + 1); - types.add(SecondaryIndex.keyComparator); - for (int i = 0; i < ckCount; i++) - types.add(baseMetadata.comparator.subtype(i)); - return new CompoundDenseCellNameType(types); - } - - protected ByteBuffer getIndexedValue(ByteBuffer rowKey, Cell cell) + protected ByteBuffer getIndexedValue(ByteBuffer rowKey, Clustering clustering, ByteBuffer cellValue, CellPath path) { CompositeType keyComparator = (CompositeType)baseCfs.metadata.getKeyValidator(); ByteBuffer[] components = keyComparator.split(rowKey); return components[columnDef.position()]; } - protected Composite makeIndexColumnPrefix(ByteBuffer rowKey, Composite columnName) + protected CBuilder buildIndexClusteringPrefix(ByteBuffer rowKey, ClusteringPrefix prefix, CellPath path) { - int count = Math.min(baseCfs.metadata.clusteringColumns().size(), columnName.size()); - CBuilder builder = getIndexComparator().prefixBuilder(); + CBuilder builder = CBuilder.create(getIndexComparator()); builder.add(rowKey); - for (int i = 0; i < count; i++) - builder.add(columnName.get(i)); - return builder.build(); + for (int i = 0; i < prefix.size(); i++) + builder.add(prefix.get(i)); + return builder; } - public IndexedEntry decodeEntry(DecoratedKey indexedValue, Cell indexEntry) + public IndexedEntry decodeEntry(DecoratedKey indexedValue, Row indexEntry) { int ckCount = baseCfs.metadata.clusteringColumns().size(); - CBuilder builder = baseCfs.getComparator().builder(); + Clustering clustering = indexEntry.clustering(); + CBuilder builder = CBuilder.create(baseCfs.getComparator()); for (int i = 0; i < ckCount; i++) - builder.add(indexEntry.name().get(i + 1)); + builder.add(clustering.get(i + 1)); - return new IndexedEntry(indexedValue, indexEntry.name(), indexEntry.timestamp(), indexEntry.name().get(0), builder.build()); + return new IndexedEntry(indexedValue, clustering, indexEntry.primaryKeyLivenessInfo().timestamp(), clustering.get(0), builder.build()); } @Override - public boolean indexes(CellName name) + protected boolean indexPrimaryKeyColumn() { - // Since a partition key is always full, we always index it return true; } - public boolean isStale(IndexedEntry entry, ColumnFamily data, long now) + @Override + public boolean indexes(ColumnDefinition c) { - return data.hasOnlyTombstones(now); + // Actual indexing for this index type is done through maybeIndex + return false; + } + + public boolean isStale(Row data, ByteBuffer indexValue, int nowInSec) + { + return !data.hasLiveData(nowInSec); } @Override - public void delete(ByteBuffer rowKey, Cell cell, OpOrder.Group opGroup) + public void maybeIndex(ByteBuffer partitionKey, Clustering clustering, long timestamp, int ttl, OpOrder.Group opGroup, int nowInSec) + { + insert(partitionKey, clustering, null, SimpleLivenessInfo.forUpdate(timestamp, ttl, nowInSec, indexCfs.metadata), opGroup); + } + + @Override + public void delete(ByteBuffer rowKey, Clustering clustering, Cell cell, OpOrder.Group opGroup, int nowInSec) { // We only know that one column of the CQL row has been updated/deleted, but we don't know if the // full row has been deleted so we should not do anything. If it ends up that the whole row has diff --git a/src/java/org/apache/cassandra/db/index/composites/CompositesIndexOnRegular.java b/src/java/org/apache/cassandra/db/index/composites/CompositesIndexOnRegular.java index b9dc07f896..a88502a1a6 100644 --- a/src/java/org/apache/cassandra/db/index/composites/CompositesIndexOnRegular.java +++ b/src/java/org/apache/cassandra/db/index/composites/CompositesIndexOnRegular.java @@ -18,15 +18,9 @@ package org.apache.cassandra.db.index.composites; import java.nio.ByteBuffer; -import java.util.ArrayList; -import java.util.List; -import org.apache.cassandra.config.CFMetaData; -import org.apache.cassandra.config.ColumnDefinition; import org.apache.cassandra.db.*; -import org.apache.cassandra.db.composites.*; -import org.apache.cassandra.db.index.SecondaryIndex; -import org.apache.cassandra.db.marshal.*; +import org.apache.cassandra.db.rows.*; /** * Index on a REGULAR column definition on a composite type. @@ -47,50 +41,35 @@ import org.apache.cassandra.db.marshal.*; */ public class CompositesIndexOnRegular extends CompositesIndex { - public static CellNameType buildIndexComparator(CFMetaData baseMetadata, ColumnDefinition columnDef) + protected ByteBuffer getIndexedValue(ByteBuffer rowKey, Clustering clustering, ByteBuffer cellValue, CellPath path) { - int prefixSize = columnDef.position(); - List> types = new ArrayList>(prefixSize + 1); - types.add(SecondaryIndex.keyComparator); - for (int i = 0; i < prefixSize; i++) - types.add(baseMetadata.comparator.subtype(i)); - return new CompoundDenseCellNameType(types); + return cellValue; } - protected ByteBuffer getIndexedValue(ByteBuffer rowKey, Cell cell) + protected CBuilder buildIndexClusteringPrefix(ByteBuffer rowKey, ClusteringPrefix prefix, CellPath path) { - return cell.value(); - } - - protected Composite makeIndexColumnPrefix(ByteBuffer rowKey, Composite cellName) - { - CBuilder builder = getIndexComparator().prefixBuilder(); + CBuilder builder = CBuilder.create(getIndexComparator()); builder.add(rowKey); - for (int i = 0; i < Math.min(columnDef.position(), cellName.size()); i++) - builder.add(cellName.get(i)); - return builder.build(); + for (int i = 0; i < prefix.size(); i++) + builder.add(prefix.get(i)); + return builder; } - public IndexedEntry decodeEntry(DecoratedKey indexedValue, Cell indexEntry) + public IndexedEntry decodeEntry(DecoratedKey indexedValue, Row indexEntry) { - CBuilder builder = baseCfs.getComparator().builder(); - for (int i = 0; i < columnDef.position(); i++) - builder.add(indexEntry.name().get(i + 1)); - return new IndexedEntry(indexedValue, indexEntry.name(), indexEntry.timestamp(), indexEntry.name().get(0), builder.build()); + Clustering clustering = indexEntry.clustering(); + ClusteringComparator baseComparator = baseCfs.getComparator(); + CBuilder builder = CBuilder.create(baseComparator); + for (int i = 0; i < baseComparator.size(); i++) + builder.add(clustering.get(i + 1)); + return new IndexedEntry(indexedValue, clustering, indexEntry.primaryKeyLivenessInfo().timestamp(), clustering.get(0), builder.build()); } - @Override - public boolean indexes(CellName name) + public boolean isStale(Row data, ByteBuffer indexValue, int nowInSec) { - AbstractType comp = baseCfs.metadata.getColumnDefinitionComparator(columnDef); - return name.size() > columnDef.position() - && comp.compare(name.get(columnDef.position()), columnDef.name.bytes) == 0; - } - - public boolean isStale(IndexedEntry entry, ColumnFamily data, long now) - { - CellName name = data.getComparator().create(entry.indexedEntryPrefix, columnDef); - Cell cell = data.getColumn(name); - return cell == null || !cell.isLive(now) || columnDef.type.compare(entry.indexValue.getKey(), cell.value()) != 0; + Cell cell = data.getCell(columnDef); + return cell == null + || !cell.isLive(nowInSec) + || columnDef.type.compare(indexValue, cell.value()) != 0; } } diff --git a/src/java/org/apache/cassandra/db/index/composites/CompositesSearcher.java b/src/java/org/apache/cassandra/db/index/composites/CompositesSearcher.java index 88453df873..f838ff1eb0 100644 --- a/src/java/org/apache/cassandra/db/index/composites/CompositesSearcher.java +++ b/src/java/org/apache/cassandra/db/index/composites/CompositesSearcher.java @@ -17,296 +17,224 @@ */ package org.apache.cassandra.db.index.composites; -import java.io.IOException; import java.nio.ByteBuffer; import java.util.*; import org.slf4j.Logger; import org.slf4j.LoggerFactory; -import org.apache.cassandra.db.ArrayBackedSortedColumns; -import org.apache.cassandra.db.Cell; -import org.apache.cassandra.db.ColumnFamily; -import org.apache.cassandra.db.ColumnFamilyStore; -import org.apache.cassandra.db.DecoratedKey; -import org.apache.cassandra.db.IndexExpression; -import org.apache.cassandra.db.Row; -import org.apache.cassandra.db.RowPosition; -import org.apache.cassandra.db.composites.CellNameType; -import org.apache.cassandra.db.composites.Composite; -import org.apache.cassandra.db.composites.Composites; -import org.apache.cassandra.db.filter.ColumnSlice; -import org.apache.cassandra.db.filter.ExtendedFilter; -import org.apache.cassandra.db.filter.IDiskAtomFilter; -import org.apache.cassandra.db.filter.QueryFilter; -import org.apache.cassandra.db.filter.SliceQueryFilter; -import org.apache.cassandra.db.index.SecondaryIndexManager; -import org.apache.cassandra.db.index.SecondaryIndexSearcher; -import org.apache.cassandra.dht.AbstractBounds; -import org.apache.cassandra.utils.ByteBufferUtil; +import org.apache.cassandra.config.ColumnDefinition; +import org.apache.cassandra.db.*; +import org.apache.cassandra.db.rows.*; +import org.apache.cassandra.db.filter.*; +import org.apache.cassandra.db.index.*; +import org.apache.cassandra.db.partitions.UnfilteredPartitionIterator; import org.apache.cassandra.utils.concurrent.OpOrder; + public class CompositesSearcher extends SecondaryIndexSearcher { private static final Logger logger = LoggerFactory.getLogger(CompositesSearcher.class); - public CompositesSearcher(SecondaryIndexManager indexManager, Set columns) + public CompositesSearcher(SecondaryIndexManager indexManager, Set columns) { super(indexManager, columns); } - @Override - public List search(ExtendedFilter filter) + private boolean isMatchingEntry(DecoratedKey partitionKey, CompositesIndex.IndexedEntry entry, ReadCommand command) { - assert filter.getClause() != null && !filter.getClause().isEmpty(); - final IndexExpression primary = highestSelectivityPredicate(filter.getClause(), true); - final CompositesIndex index = (CompositesIndex)indexManager.getIndexForColumn(primary.column); - // TODO: this should perhaps not open and maintain a writeOp for the full duration, but instead only *try* to delete stale entries, without blocking if there's no room - // as it stands, we open a writeOp and keep it open for the duration to ensure that should this CF get flushed to make room we don't block the reclamation of any room being made - try (OpOrder.Group writeOp = baseCfs.keyspace.writeOrder.start(); OpOrder.Group baseOp = baseCfs.readOrdering.start(); OpOrder.Group indexOp = index.getIndexCfs().readOrdering.start()) - { - return baseCfs.filter(getIndexedIterator(writeOp, filter, primary, index), filter); - } + return command.selects(partitionKey, entry.indexedEntryClustering); } - private Composite makePrefix(CompositesIndex index, ByteBuffer key, ExtendedFilter filter, boolean isStart) + protected UnfilteredPartitionIterator queryDataFromIndex(AbstractSimplePerColumnSecondaryIndex secondaryIdx, + final DecoratedKey indexKey, + final RowIterator indexHits, + final ReadCommand command, + final ReadOrderGroup orderGroup) { - if (key.remaining() == 0) - return Composites.EMPTY; + assert indexHits.staticRow() == Rows.EMPTY_STATIC_ROW; - Composite prefix; - IDiskAtomFilter columnFilter = filter.columnFilter(key); - if (columnFilter instanceof SliceQueryFilter) + assert secondaryIdx instanceof CompositesIndex; + final CompositesIndex index = (CompositesIndex)secondaryIdx; + + return new UnfilteredPartitionIterator() { - SliceQueryFilter sqf = (SliceQueryFilter)columnFilter; - Composite columnName = isStart ? sqf.start() : sqf.finish(); - prefix = columnName.isEmpty() ? index.getIndexComparator().make(key) : index.makeIndexColumnPrefix(key, columnName); - } - else - { - prefix = index.getIndexComparator().make(key); - } - return isStart ? prefix.start() : prefix.end(); - } + private CompositesIndex.IndexedEntry nextEntry; - private ColumnFamilyStore.AbstractScanIterator getIndexedIterator(final OpOrder.Group writeOp, final ExtendedFilter filter, final IndexExpression primary, final CompositesIndex index) - { - // Start with the most-restrictive indexed clause, then apply remaining clauses - // to each row matching that clause. - // TODO: allow merge join instead of just one index + loop - assert index != null; - assert index.getIndexCfs() != null; - final DecoratedKey indexKey = index.getIndexKeyFor(primary.value); + private UnfilteredRowIterator next; - if (logger.isDebugEnabled()) - logger.debug("Most-selective indexed predicate is {}", index.expressionString(primary)); - - /* - * XXX: If the range requested is a token range, we'll have to start at the beginning (and stop at the end) of - * the indexed row unfortunately (which will be inefficient), because we have not way to intuit the smallest - * possible key having a given token. A fix would be to actually store the token along the key in the - * indexed row. - */ - final AbstractBounds range = filter.dataRange.keyRange(); - ByteBuffer startKey = range.left instanceof DecoratedKey ? ((DecoratedKey)range.left).getKey() : ByteBufferUtil.EMPTY_BYTE_BUFFER; - ByteBuffer endKey = range.right instanceof DecoratedKey ? ((DecoratedKey)range.right).getKey() : ByteBufferUtil.EMPTY_BYTE_BUFFER; - - final CellNameType baseComparator = baseCfs.getComparator(); - final CellNameType indexComparator = index.getIndexCfs().getComparator(); - - final Composite startPrefix = makePrefix(index, startKey, filter, true); - final Composite endPrefix = makePrefix(index, endKey, filter, false); - - return new ColumnFamilyStore.AbstractScanIterator() - { - private Composite lastSeenPrefix = startPrefix; - private Deque indexCells; - private int columnsRead = Integer.MAX_VALUE; - private int limit = filter.currentLimit(); - private int columnsCount = 0; - - // We have to fetch at least two rows to avoid breaking paging if the first row doesn't satisfy all clauses - private int indexCellsPerQuery = Math.max(2, Math.min(filter.maxColumns(), filter.maxRows())); - - public boolean needsFiltering() + public boolean isForThrift() { + return command.isForThrift(); + } + + public boolean hasNext() + { + return prepareNext(); + } + + public UnfilteredRowIterator next() + { + if (next == null) + prepareNext(); + + UnfilteredRowIterator toReturn = next; + next = null; + return toReturn; + } + + private boolean prepareNext() + { + if (next != null) + return true; + + if (nextEntry == null) + { + if (!indexHits.hasNext()) + return false; + + nextEntry = index.decodeEntry(indexKey, indexHits.next()); + } + + // Gather all index hits belonging to the same partition and query the data for those hits. + // TODO: it's much more efficient to do 1 read for all hits to the same partition than doing + // 1 read per index hit. However, this basically mean materializing all hits for a partition + // in memory so we should consider adding some paging mechanism. However, index hits should + // be relatively small so it's much better than the previous code that was materializing all + // *data* for a given partition. + NavigableSet clusterings = new TreeSet<>(baseCfs.getComparator()); + List entries = new ArrayList<>(); + DecoratedKey partitionKey = baseCfs.partitioner.decorateKey(nextEntry.indexedKey); + + while (nextEntry != null && partitionKey.getKey().equals(nextEntry.indexedKey)) + { + // We're queried a slice of the index, but some hits may not match some of the clustering column constraints + if (isMatchingEntry(partitionKey, nextEntry, command)) + { + clusterings.add(nextEntry.indexedEntryClustering); + entries.add(nextEntry); + } + + nextEntry = indexHits.hasNext() ? index.decodeEntry(indexKey, indexHits.next()) : null; + } + + // Because we've eliminated entries that don't match the clustering columns, it's possible we added nothing + if (clusterings.isEmpty()) + return prepareNext(); + + // Query the gathered index hits. We still need to filter stale hits from the resulting query. + ClusteringIndexNamesFilter filter = new ClusteringIndexNamesFilter(clusterings, false); + SinglePartitionReadCommand dataCmd = new SinglePartitionNamesCommand(baseCfs.metadata, + command.nowInSec(), + command.columnFilter(), + command.rowFilter(), + DataLimits.NONE, + partitionKey, + filter); + @SuppressWarnings("resource") // We close right away if empty, and if it's assign to next it will be called either + // by the next caller of next, or through closing this iterator is this come before. + UnfilteredRowIterator dataIter = filterStaleEntries(dataCmd.queryMemtableAndDisk(baseCfs, orderGroup.baseReadOpOrderGroup()), + index, + indexKey.getKey(), + entries, + orderGroup.writeOpOrderGroup(), + command.nowInSec()); + if (dataIter.isEmpty()) + { + dataIter.close(); + return prepareNext(); + } + + next = dataIter; + return true; + } + + public void remove() + { + throw new UnsupportedOperationException(); + } + + public void close() + { + indexHits.close(); + if (next != null) + next.close(); + } + }; + } + + private UnfilteredRowIterator filterStaleEntries(UnfilteredRowIterator dataIter, + final CompositesIndex index, + final ByteBuffer indexValue, + final List entries, + final OpOrder.Group writeOp, + final int nowInSec) + { + return new WrappingUnfilteredRowIterator(dataIter) + { + private int entriesIdx; + private Unfiltered next; + + @Override + public boolean hasNext() + { + return prepareNext(); + } + + @Override + public Unfiltered next() + { + if (next == null) + prepareNext(); + + Unfiltered toReturn = next; + next = null; + return toReturn; + } + + private boolean prepareNext() + { + if (next != null) + return true; + + while (next == null && super.hasNext()) + { + next = super.next(); + if (next.kind() != Unfiltered.Kind.ROW) + return true; + + Row row = (Row)next; + CompositesIndex.IndexedEntry entry = findEntry(row.clustering(), writeOp, nowInSec); + if (!index.isStale(row, indexValue, nowInSec)) + return true; + + // The entry is stale: delete the entry and ignore otherwise + index.delete(entry, writeOp, nowInSec); + next = null; + } return false; } - private Row makeReturn(DecoratedKey key, ColumnFamily data) + private CompositesIndex.IndexedEntry findEntry(Clustering clustering, OpOrder.Group writeOp, int nowInSec) { - if (data == null) - return endOfData(); - - assert key != null; - return new Row(key, data); - } - - protected Row computeNext() - { - /* - * Our internal index code is wired toward internal rows. So we need to accumulate all results for a given - * row before returning from this method. Which unfortunately means that this method has to do what - * CFS.filter does for KeysIndex. - */ - DecoratedKey currentKey = null; - ColumnFamily data = null; - Composite previousPrefix = null; - - while (true) + assert entriesIdx < entries.size(); + while (entriesIdx < entries.size()) { - // Did we get more columns that needed to respect the user limit? - // (but we still need to return what has been fetched already) - if (columnsCount >= limit) - return makeReturn(currentKey, data); - - if (indexCells == null || indexCells.isEmpty()) - { - if (columnsRead < indexCellsPerQuery) - { - logger.trace("Read only {} (< {}) last page through, must be done", columnsRead, indexCellsPerQuery); - return makeReturn(currentKey, data); - } - - if (logger.isTraceEnabled()) - logger.trace("Scanning index {} starting with {}", - index.expressionString(primary), indexComparator.getString(startPrefix)); - - QueryFilter indexFilter = QueryFilter.getSliceFilter(indexKey, - index.getIndexCfs().name, - lastSeenPrefix, - endPrefix, - false, - indexCellsPerQuery, - filter.timestamp); - ColumnFamily indexRow = index.getIndexCfs().getColumnFamily(indexFilter); - if (indexRow == null || !indexRow.hasColumns()) - return makeReturn(currentKey, data); - - Collection sortedCells = indexRow.getSortedColumns(); - columnsRead = sortedCells.size(); - indexCells = new ArrayDeque<>(sortedCells); - Cell firstCell = sortedCells.iterator().next(); - - // Paging is racy, so it is possible the first column of a page is not the last seen one. - if (lastSeenPrefix != startPrefix && lastSeenPrefix.equals(firstCell.name())) - { - // skip the row we already saw w/ the last page of results - indexCells.poll(); - logger.trace("Skipping {}", indexComparator.getString(firstCell.name())); - } - } - - while (!indexCells.isEmpty() && columnsCount <= limit) - { - Cell cell = indexCells.poll(); - lastSeenPrefix = cell.name(); - if (!cell.isLive(filter.timestamp)) - { - logger.trace("skipping {}", cell.name()); - continue; - } - - CompositesIndex.IndexedEntry entry = index.decodeEntry(indexKey, cell); - DecoratedKey dk = baseCfs.partitioner.decorateKey(entry.indexedKey); - - // Are we done for this row? - if (currentKey == null) - { - currentKey = dk; - } - else if (!currentKey.equals(dk)) - { - DecoratedKey previousKey = currentKey; - currentKey = dk; - previousPrefix = null; - - // We're done with the previous row, return it if it had data, continue otherwise - indexCells.addFirst(cell); - if (data == null) - continue; - else - return makeReturn(previousKey, data); - } - - if (!range.contains(dk)) - { - // Either we're not yet in the range cause the range is start excluding, or we're - // past it. - if (!range.right.isMinimum() && range.right.compareTo(dk) < 0) - { - logger.trace("Reached end of assigned scan range"); - return endOfData(); - } - else - { - logger.debug("Skipping entry {} before assigned scan range", dk.getToken()); - continue; - } - } - - // Check if this entry cannot be a hit due to the original cell filter - Composite start = entry.indexedEntryPrefix; - if (!filter.columnFilter(dk.getKey()).maySelectPrefix(baseComparator, start)) - continue; - - // If we've record the previous prefix, it means we're dealing with an index on the collection value. In - // that case, we can have multiple index prefix for the same CQL3 row. In that case, we want to only add - // the CQL3 row once (because requesting the data multiple time would be inefficient but more importantly - // because we shouldn't count the columns multiple times with the lastCounted() call at the end of this - // method). - if (previousPrefix != null && previousPrefix.equals(start)) - continue; - else - previousPrefix = null; - - if (logger.isTraceEnabled()) - logger.trace("Adding index hit to current row for {}", indexComparator.getString(cell.name())); - - // We always query the whole CQL3 row. In the case where the original filter was a name filter this might be - // slightly wasteful, but this probably doesn't matter in practice and it simplify things. - ColumnSlice dataSlice = new ColumnSlice(start, entry.indexedEntryPrefix.end()); - // If the table has static columns, we must fetch them too as they may need to be returned too. - // Note that this is potentially wasteful for 2 reasons: - // 1) we will retrieve the static parts for each indexed row, even if we have more than one row in - // the same partition. If we were to group data queries to rows on the same slice, which would - // speed up things in general, we would also optimize here since we would fetch static columns only - // once for each group. - // 2) at this point we don't know if the user asked for static columns or not, so we might be fetching - // them for nothing. We would however need to ship the list of "CQL3 columns selected" with getRangeSlice - // to be able to know that. - // TODO: we should improve both point above - ColumnSlice[] slices = baseCfs.metadata.hasStaticColumns() - ? new ColumnSlice[]{ baseCfs.metadata.comparator.staticPrefix().slice(), dataSlice } - : new ColumnSlice[]{ dataSlice }; - SliceQueryFilter dataFilter = new SliceQueryFilter(slices, false, Integer.MAX_VALUE, baseCfs.metadata.clusteringColumns().size()); - ColumnFamily newData = baseCfs.getColumnFamily(new QueryFilter(dk, baseCfs.name, dataFilter, filter.timestamp)); - if (newData == null || index.isStale(entry, newData, filter.timestamp)) - { - index.delete(entry, writeOp); - continue; - } - - assert newData != null : "An entry with no data should have been considered stale"; - - // We know the entry is not stale and so the entry satisfy the primary clause. So whether - // or not the data satisfies the other clauses, there will be no point to re-check the - // same CQL3 row if we run into another collection value entry for this row. - if (entry.indexedEntryCollectionKey != null) - previousPrefix = start; - - if (!filter.isSatisfiedBy(dk, newData, entry.indexedEntryPrefix, entry.indexedEntryCollectionKey)) - continue; - - if (data == null) - data = ArrayBackedSortedColumns.factory.create(baseCfs.metadata); - data.addAll(newData); - columnsCount += dataFilter.lastCounted(); - } - } - } - - public void close() throws IOException {} + CompositesIndex.IndexedEntry entry = entries.get(entriesIdx++); + // The entries are in clustering order. So that the requested entry should be the + // next entry, the one at 'entriesIdx'. However, we can have stale entries, entries + // that have no corresponding row in the base table typically because of a range + // tombstone or partition level deletion. Delete such stale entries. + int cmp = metadata().comparator.compare(entry.indexedEntryClustering, clustering); + assert cmp <= 0; // this would means entries are not in clustering order, which shouldn't happen + if (cmp == 0) + return entry; + else + index.delete(entry, writeOp, nowInSec); + } + // entries correspond to the rows we've queried, so we shouldn't have a row that has no corresponding entry. + throw new AssertionError(); + } }; } } diff --git a/src/java/org/apache/cassandra/db/index/keys/KeysIndex.java b/src/java/org/apache/cassandra/db/index/keys/KeysIndex.java index e771d991d0..7930bd6f9d 100644 --- a/src/java/org/apache/cassandra/db/index/keys/KeysIndex.java +++ b/src/java/org/apache/cassandra/db/index/keys/KeysIndex.java @@ -20,14 +20,15 @@ package org.apache.cassandra.db.index.keys; import java.nio.ByteBuffer; import java.util.Set; -import org.apache.cassandra.db.Cell; -import org.apache.cassandra.db.composites.CellName; -import org.apache.cassandra.db.composites.CellNames; -import org.apache.cassandra.db.ColumnFamily; +import org.apache.cassandra.config.CFMetaData; +import org.apache.cassandra.config.ColumnDefinition; +import org.apache.cassandra.db.*; +import org.apache.cassandra.db.index.SecondaryIndex; +import org.apache.cassandra.db.rows.*; import org.apache.cassandra.db.index.AbstractSimplePerColumnSecondaryIndex; import org.apache.cassandra.db.index.SecondaryIndexSearcher; -import org.apache.cassandra.db.marshal.AbstractType; import org.apache.cassandra.exceptions.ConfigurationException; +import org.apache.cassandra.utils.concurrent.OpOrder; /** * Implements a secondary index for a column family using a second column family. @@ -39,41 +40,51 @@ import org.apache.cassandra.exceptions.ConfigurationException; */ public class KeysIndex extends AbstractSimplePerColumnSecondaryIndex { - protected ByteBuffer getIndexedValue(ByteBuffer rowKey, Cell cell) + public static void addIndexClusteringColumns(CFMetaData.Builder indexMetadata, CFMetaData baseMetadata, ColumnDefinition cfDef) { - return cell.value(); + indexMetadata.addClusteringColumn("partition_key", SecondaryIndex.keyComparator); } - protected CellName makeIndexColumnName(ByteBuffer rowKey, Cell cell) + @Override + public void indexRow(DecoratedKey key, Row row, OpOrder.Group opGroup, int nowInSec) { - return CellNames.simpleDense(rowKey); + super.indexRow(key, row, opGroup, nowInSec); + + // This is used when building indexes, in particular when the index is first created. On thrift, this + // potentially means the column definition just got created, and so we need to check if's not a "dynamic" + // row that actually correspond to the index definition. + assert baseCfs.metadata.isCompactTable(); + if (!row.isStatic()) + { + Clustering clustering = row.clustering(); + if (clustering.get(0).equals(columnDef.name.bytes)) + { + Cell cell = row.getCell(baseCfs.metadata.compactValueColumn()); + if (cell != null && cell.isLive(nowInSec)) + insert(key.getKey(), clustering, cell, opGroup); + } + } } - public SecondaryIndexSearcher createSecondaryIndexSearcher(Set columns) + protected ByteBuffer getIndexedValue(ByteBuffer rowKey, Clustering clustering, ByteBuffer cellValue, CellPath path) + { + return cellValue; + } + + protected CBuilder buildIndexClusteringPrefix(ByteBuffer rowKey, ClusteringPrefix prefix, CellPath path) + { + CBuilder builder = CBuilder.create(getIndexComparator()); + builder.add(rowKey); + return builder; + } + + public SecondaryIndexSearcher createSecondaryIndexSearcher(Set columns) { return new KeysSearcher(baseCfs.indexManager, columns); } - public boolean isIndexEntryStale(ByteBuffer indexedValue, ColumnFamily data, long now) - { - Cell cell = data.getColumn(data.getComparator().makeCellName(columnDef.name.bytes)); - return cell == null || !cell.isLive(now) || columnDef.type.compare(indexedValue, cell.value()) != 0; - } - public void validateOptions() throws ConfigurationException { // no options used } - - public boolean indexes(CellName name) - { - // This consider the full cellName directly - AbstractType comparator = baseCfs.metadata.getColumnDefinitionComparator(columnDef); - return comparator.compare(columnDef.name.bytes, name.toByteBuffer()) == 0; - } - - protected AbstractType getExpressionComparator() - { - return baseCfs.getComparator().asAbstractType(); - } } diff --git a/src/java/org/apache/cassandra/db/index/keys/KeysSearcher.java b/src/java/org/apache/cassandra/db/index/keys/KeysSearcher.java index b4fd0bae55..6b536404cd 100644 --- a/src/java/org/apache/cassandra/db/index/keys/KeysSearcher.java +++ b/src/java/org/apache/cassandra/db/index/keys/KeysSearcher.java @@ -17,190 +17,169 @@ */ package org.apache.cassandra.db.index.keys; -import java.io.IOException; import java.nio.ByteBuffer; -import java.util.Collection; -import java.util.Iterator; -import java.util.List; import java.util.Set; import org.slf4j.Logger; import org.slf4j.LoggerFactory; +import org.apache.cassandra.config.ColumnDefinition; import org.apache.cassandra.db.*; -import org.apache.cassandra.db.composites.CellName; -import org.apache.cassandra.db.composites.CellNameType; -import org.apache.cassandra.db.composites.Composite; -import org.apache.cassandra.db.composites.Composites; -import org.apache.cassandra.db.filter.ExtendedFilter; -import org.apache.cassandra.db.filter.IDiskAtomFilter; -import org.apache.cassandra.db.filter.QueryFilter; +import org.apache.cassandra.db.rows.*; +import org.apache.cassandra.db.partitions.*; import org.apache.cassandra.db.index.*; -import org.apache.cassandra.dht.AbstractBounds; -import org.apache.cassandra.dht.Range; +import org.apache.cassandra.db.filter.*; +import org.apache.cassandra.db.partitions.UnfilteredPartitionIterator; import org.apache.cassandra.utils.concurrent.OpOrder; public class KeysSearcher extends SecondaryIndexSearcher { private static final Logger logger = LoggerFactory.getLogger(KeysSearcher.class); - public KeysSearcher(SecondaryIndexManager indexManager, Set columns) + public KeysSearcher(SecondaryIndexManager indexManager, Set columns) { super(indexManager, columns); } - @Override - public List search(ExtendedFilter filter) + protected UnfilteredPartitionIterator queryDataFromIndex(final AbstractSimplePerColumnSecondaryIndex index, + final DecoratedKey indexKey, + final RowIterator indexHits, + final ReadCommand command, + final ReadOrderGroup orderGroup) { - assert filter.getClause() != null && !filter.getClause().isEmpty(); - final IndexExpression primary = highestSelectivityPredicate(filter.getClause(), true); - final SecondaryIndex index = indexManager.getIndexForColumn(primary.column); - // TODO: this should perhaps not open and maintain a writeOp for the full duration, but instead only *try* to delete stale entries, without blocking if there's no room - // as it stands, we open a writeOp and keep it open for the duration to ensure that should this CF get flushed to make room we don't block the reclamation of any room being made - try (OpOrder.Group writeOp = baseCfs.keyspace.writeOrder.start(); OpOrder.Group baseOp = baseCfs.readOrdering.start(); OpOrder.Group indexOp = index.getIndexCfs().readOrdering.start()) + assert indexHits.staticRow() == Rows.EMPTY_STATIC_ROW; + + return new UnfilteredPartitionIterator() { - return baseCfs.filter(getIndexedIterator(writeOp, filter, primary, index), filter); + private UnfilteredRowIterator next; + + public boolean isForThrift() + { + return command.isForThrift(); + } + + public boolean hasNext() + { + return prepareNext(); + } + + public UnfilteredRowIterator next() + { + if (next == null) + prepareNext(); + + UnfilteredRowIterator toReturn = next; + next = null; + return toReturn; + } + + private boolean prepareNext() + { + while (next == null && indexHits.hasNext()) + { + Row hit = indexHits.next(); + DecoratedKey key = baseCfs.partitioner.decorateKey(hit.clustering().get(0)); + + SinglePartitionReadCommand dataCmd = SinglePartitionReadCommand.create(isForThrift(), + baseCfs.metadata, + command.nowInSec(), + command.columnFilter(), + command.rowFilter(), + DataLimits.NONE, + key, + command.clusteringIndexFilter(key)); + @SuppressWarnings("resource") // filterIfStale closes it's iterator if either it materialize it or if it returns null. + // Otherwise, we close right away if empty, and if it's assigned to next it will be called either + // by the next caller of next, or through closing this iterator is this come before. + UnfilteredRowIterator dataIter = filterIfStale(dataCmd.queryMemtableAndDisk(baseCfs, orderGroup.baseReadOpOrderGroup()), + index, + hit, + indexKey.getKey(), + orderGroup.writeOpOrderGroup(), + isForThrift(), + command.nowInSec()); + + if (dataIter != null) + { + if (dataIter.isEmpty()) + dataIter.close(); + else + next = dataIter; + } + } + return next != null; + } + + public void remove() + { + throw new UnsupportedOperationException(); + } + + public void close() + { + indexHits.close(); + if (next != null) + next.close(); + } + }; + } + + private UnfilteredRowIterator filterIfStale(UnfilteredRowIterator iterator, + AbstractSimplePerColumnSecondaryIndex index, + Row indexHit, + ByteBuffer indexedValue, + OpOrder.Group writeOp, + boolean isForThrift, + int nowInSec) + { + if (isForThrift) + { + // The data we got has gone though ThrifResultsMerger, so we're looking for the row whose clustering + // is the indexed name. Ans so we need to materialize the partition. + ArrayBackedPartition result = ArrayBackedPartition.create(iterator); + iterator.close(); + Row data = result.getRow(new SimpleClustering(index.indexedColumn().name.bytes)); + Cell cell = data == null ? null : data.getCell(baseCfs.metadata.compactValueColumn()); + return deleteIfStale(iterator.partitionKey(), cell, index, indexHit, indexedValue, writeOp, nowInSec) + ? null + : result.unfilteredIterator(); + } + else + { + assert iterator.metadata().isCompactTable(); + Row data = iterator.staticRow(); + Cell cell = data == null ? null : data.getCell(index.indexedColumn()); + if (deleteIfStale(iterator.partitionKey(), cell, index, indexHit, indexedValue, writeOp, nowInSec)) + { + iterator.close(); + return null; + } + else + { + return iterator; + } } } - private ColumnFamilyStore.AbstractScanIterator getIndexedIterator(final OpOrder.Group writeOp, final ExtendedFilter filter, final IndexExpression primary, final SecondaryIndex index) + private boolean deleteIfStale(DecoratedKey partitionKey, + Cell cell, + AbstractSimplePerColumnSecondaryIndex index, + Row indexHit, + ByteBuffer indexedValue, + OpOrder.Group writeOp, + int nowInSec) { - - // Start with the most-restrictive indexed clause, then apply remaining clauses - // to each row matching that clause. - // TODO: allow merge join instead of just one index + loop - assert index != null; - assert index.getIndexCfs() != null; - final DecoratedKey indexKey = index.getIndexKeyFor(primary.value); - - if (logger.isDebugEnabled()) - logger.debug("Most-selective indexed predicate is {}", - ((AbstractSimplePerColumnSecondaryIndex) index).expressionString(primary)); - - /* - * XXX: If the range requested is a token range, we'll have to start at the beginning (and stop at the end) of - * the indexed row unfortunately (which will be inefficient), because we have not way to intuit the small - * possible key having a given token. A fix would be to actually store the token along the key in the - * indexed row. - */ - final AbstractBounds range = filter.dataRange.keyRange(); - CellNameType type = index.getIndexCfs().getComparator(); - final Composite startKey = range.left instanceof DecoratedKey ? type.make(((DecoratedKey)range.left).getKey()) : Composites.EMPTY; - final Composite endKey = range.right instanceof DecoratedKey ? type.make(((DecoratedKey)range.right).getKey()) : Composites.EMPTY; - - final CellName primaryColumn = baseCfs.getComparator().cellFromByteBuffer(primary.column); - - return new ColumnFamilyStore.AbstractScanIterator() + if (cell == null || !cell.isLive(nowInSec) || index.indexedColumn().type.compare(indexedValue, cell.value()) != 0) { - private Composite lastSeenKey = startKey; - private Iterator indexColumns; - private int columnsRead = Integer.MAX_VALUE; - - protected Row computeNext() - { - int meanColumns = Math.max(index.getIndexCfs().getMeanColumns(), 1); - // We shouldn't fetch only 1 row as this provides buggy paging in case the first row doesn't satisfy all clauses - int rowsPerQuery = Math.max(Math.min(filter.maxRows(), filter.maxColumns() / meanColumns), 2); - while (true) - { - if (indexColumns == null || !indexColumns.hasNext()) - { - if (columnsRead < rowsPerQuery) - { - logger.trace("Read only {} (< {}) last page through, must be done", columnsRead, rowsPerQuery); - return endOfData(); - } - - if (logger.isTraceEnabled() && (index instanceof AbstractSimplePerColumnSecondaryIndex)) - logger.trace("Scanning index {} starting with {}", - ((AbstractSimplePerColumnSecondaryIndex)index).expressionString(primary), index.getBaseCfs().metadata.getKeyValidator().getString(startKey.toByteBuffer())); - - QueryFilter indexFilter = QueryFilter.getSliceFilter(indexKey, - index.getIndexCfs().name, - lastSeenKey, - endKey, - false, - rowsPerQuery, - filter.timestamp); - ColumnFamily indexRow = index.getIndexCfs().getColumnFamily(indexFilter); - logger.trace("fetched {}", indexRow); - if (indexRow == null) - { - logger.trace("no data, all done"); - return endOfData(); - } - - Collection sortedCells = indexRow.getSortedColumns(); - columnsRead = sortedCells.size(); - indexColumns = sortedCells.iterator(); - Cell firstCell = sortedCells.iterator().next(); - - // Paging is racy, so it is possible the first column of a page is not the last seen one. - if (lastSeenKey != startKey && lastSeenKey.equals(firstCell.name())) - { - // skip the row we already saw w/ the last page of results - indexColumns.next(); - logger.trace("Skipping {}", baseCfs.metadata.getKeyValidator().getString(firstCell.name().toByteBuffer())); - } - else if (range instanceof Range && indexColumns.hasNext() && firstCell.name().equals(startKey)) - { - // skip key excluded by range - indexColumns.next(); - logger.trace("Skipping first key as range excludes it"); - } - } - - while (indexColumns.hasNext()) - { - Cell cell = indexColumns.next(); - lastSeenKey = cell.name(); - if (!cell.isLive(filter.timestamp)) - { - logger.trace("skipping {}", cell.name()); - continue; - } - - DecoratedKey dk = baseCfs.partitioner.decorateKey(lastSeenKey.toByteBuffer()); - if (!range.right.isMinimum() && range.right.compareTo(dk) < 0) - { - logger.trace("Reached end of assigned scan range"); - return endOfData(); - } - if (!range.contains(dk)) - { - logger.trace("Skipping entry {} outside of assigned scan range", dk.getToken()); - continue; - } - - logger.trace("Returning index hit for {}", dk); - ColumnFamily data = baseCfs.getColumnFamily(new QueryFilter(dk, baseCfs.name, filter.columnFilter(lastSeenKey.toByteBuffer()), filter.timestamp)); - // While the column family we'll get in the end should contains the primary clause cell, the initialFilter may not have found it and can thus be null - if (data == null) - data = ArrayBackedSortedColumns.factory.create(baseCfs.metadata); - - // as in CFS.filter - extend the filter to ensure we include the columns - // from the index expressions, just in case they weren't included in the initialFilter - IDiskAtomFilter extraFilter = filter.getExtraFilter(dk, data); - if (extraFilter != null) - { - ColumnFamily cf = baseCfs.getColumnFamily(new QueryFilter(dk, baseCfs.name, extraFilter, filter.timestamp)); - if (cf != null) - data.addAll(cf); - } - - if (((KeysIndex)index).isIndexEntryStale(indexKey.getKey(), data, filter.timestamp)) - { - // delete the index entry w/ its own timestamp - Cell dummyCell = new BufferCell(primaryColumn, indexKey.getKey(), cell.timestamp()); - ((PerColumnSecondaryIndex)index).delete(dk.getKey(), dummyCell, writeOp); - continue; - } - return new Row(dk, data); - } - } - } - - public void close() throws IOException {} - }; + // Index is stale, remove the index entry and ignore + index.delete(partitionKey.getKey(), + new SimpleClustering(index.indexedColumn().name.bytes), + indexedValue, + null, + new SimpleDeletionTime(indexHit.primaryKeyLivenessInfo().timestamp(), nowInSec), + writeOp); + return true; + } + return false; } } diff --git a/src/java/org/apache/cassandra/db/lifecycle/SSTableIntervalTree.java b/src/java/org/apache/cassandra/db/lifecycle/SSTableIntervalTree.java index ff2abcbe5b..52ac227a69 100644 --- a/src/java/org/apache/cassandra/db/lifecycle/SSTableIntervalTree.java +++ b/src/java/org/apache/cassandra/db/lifecycle/SSTableIntervalTree.java @@ -6,16 +6,16 @@ import java.util.List; import com.google.common.collect.Iterables; -import org.apache.cassandra.db.RowPosition; +import org.apache.cassandra.db.PartitionPosition; import org.apache.cassandra.io.sstable.format.SSTableReader; import org.apache.cassandra.utils.Interval; import org.apache.cassandra.utils.IntervalTree; -public class SSTableIntervalTree extends IntervalTree> +public class SSTableIntervalTree extends IntervalTree> { private static final SSTableIntervalTree EMPTY = new SSTableIntervalTree(null); - SSTableIntervalTree(Collection> intervals) + SSTableIntervalTree(Collection> intervals) { super(intervals); } @@ -30,11 +30,11 @@ public class SSTableIntervalTree extends IntervalTree> buildIntervals(Iterable sstables) + public static List> buildIntervals(Iterable sstables) { - List> intervals = new ArrayList<>(Iterables.size(sstables)); + List> intervals = new ArrayList<>(Iterables.size(sstables)); for (SSTableReader sstable : sstables) - intervals.add(Interval.create(sstable.first, sstable.last, sstable)); + intervals.add(Interval.create(sstable.first, sstable.last, sstable)); return intervals; } -} \ No newline at end of file +} diff --git a/src/java/org/apache/cassandra/db/lifecycle/View.java b/src/java/org/apache/cassandra/db/lifecycle/View.java index 0d1100bfbc..f710dda23d 100644 --- a/src/java/org/apache/cassandra/db/lifecycle/View.java +++ b/src/java/org/apache/cassandra/db/lifecycle/View.java @@ -25,7 +25,7 @@ import com.google.common.base.Predicate; import com.google.common.collect.*; import org.apache.cassandra.db.Memtable; -import org.apache.cassandra.db.RowPosition; +import org.apache.cassandra.db.PartitionPosition; import org.apache.cassandra.dht.AbstractBounds; import org.apache.cassandra.io.sstable.format.SSTableReader; import org.apache.cassandra.utils.Interval; @@ -126,12 +126,12 @@ public class View return String.format("View(pending_count=%d, sstables=%s, compacting=%s)", liveMemtables.size() + flushingMemtables.size() - 1, sstables, compacting); } - public List sstablesInBounds(AbstractBounds rowBounds) + public List sstablesInBounds(AbstractBounds rowBounds) { if (intervalTree.isEmpty()) return Collections.emptyList(); - RowPosition stopInTree = rowBounds.right.isMinimum() ? intervalTree.max() : rowBounds.right; - return intervalTree.search(Interval.create(rowBounds.left, stopInTree)); + PartitionPosition stopInTree = rowBounds.right.isMinimum() ? intervalTree.max() : rowBounds.right; + return intervalTree.search(Interval.create(rowBounds.left, stopInTree)); } // METHODS TO CONSTRUCT FUNCTIONS FOR MODIFYING A VIEW: diff --git a/src/java/org/apache/cassandra/db/marshal/AbstractType.java b/src/java/org/apache/cassandra/db/marshal/AbstractType.java index aa25a81dd6..b074b34f2e 100644 --- a/src/java/org/apache/cassandra/db/marshal/AbstractType.java +++ b/src/java/org/apache/cassandra/db/marshal/AbstractType.java @@ -17,6 +17,8 @@ */ package org.apache.cassandra.db.marshal; +import java.io.DataInput; +import java.io.IOException; import java.nio.ByteBuffer; import java.util.ArrayList; import java.util.Collection; @@ -27,11 +29,15 @@ import java.util.Map; import org.apache.cassandra.cql3.CQL3Type; import org.apache.cassandra.cql3.Term; +import org.apache.cassandra.db.TypeSizes; import org.apache.cassandra.exceptions.SyntaxException; import org.apache.cassandra.serializers.TypeSerializer; import org.apache.cassandra.serializers.MarshalException; import org.github.jamm.Unmetered; +import org.apache.cassandra.io.util.DataOutputPlus; +import org.apache.cassandra.io.util.FileUtils; +import org.apache.cassandra.utils.ByteBufferUtil; /** * Specifies a Comparator for a specific type of ByteBuffer. @@ -87,6 +93,9 @@ public abstract class AbstractType implements Comparator /** get a string representation of the bytes suitable for log messages */ public String getString(ByteBuffer bytes) { + if (bytes == null) + return "null"; + TypeSerializer serializer = getSerializer(); serializer.validate(bytes); @@ -132,6 +141,17 @@ public abstract class AbstractType implements Comparator return new CQL3Type.Custom(this); } + /** + * Same as compare except that this ignore ReversedType. This is to be use when + * comparing 2 values to decide for a CQL condition (see Operator.isSatisfiedBy) as + * for CQL, ReversedType is simply an "hint" to the storage engine but it does not + * change the meaning of queries per-se. + */ + public int compareForCQL(ByteBuffer v1, ByteBuffer v2) + { + return compare(v1, v2); + } + public abstract TypeSerializer getSerializer(); /* convenience method */ @@ -290,6 +310,50 @@ public abstract class AbstractType implements Comparator return Collections.>singletonList(this); } + /** + * The length of values for this type if all values are of fixed length, -1 otherwise. + */ + protected int valueLengthIfFixed() + { + return -1; + } + + // This assumes that no empty values are passed + public void writeValue(ByteBuffer value, DataOutputPlus out) throws IOException + { + assert value.hasRemaining(); + if (valueLengthIfFixed() >= 0) + out.write(value); + else + ByteBufferUtil.writeWithLength(value, out); + } + + public long writtenLength(ByteBuffer value, TypeSizes sizes) + { + assert value.hasRemaining(); + return valueLengthIfFixed() >= 0 + ? value.remaining() + : sizes.sizeofWithLength(value); + } + + public ByteBuffer readValue(DataInput in) throws IOException + { + int length = valueLengthIfFixed(); + if (length >= 0) + return ByteBufferUtil.read(in, length); + else + return ByteBufferUtil.readWithLength(in); + } + + public void skipValue(DataInput in) throws IOException + { + int length = valueLengthIfFixed(); + if (length < 0) + length = in.readInt(); + + FileUtils.skipBytesFully(in, length); + } + /** * This must be overriden by subclasses if necessary so that for any * AbstractType, this == TypeParser.parse(toString()). diff --git a/src/java/org/apache/cassandra/db/marshal/BooleanType.java b/src/java/org/apache/cassandra/db/marshal/BooleanType.java index bfe8c346a0..f87eb121b4 100644 --- a/src/java/org/apache/cassandra/db/marshal/BooleanType.java +++ b/src/java/org/apache/cassandra/db/marshal/BooleanType.java @@ -94,4 +94,10 @@ public class BooleanType extends AbstractType { return BooleanSerializer.instance; } + + @Override + protected int valueLengthIfFixed() + { + return 1; + } } diff --git a/src/java/org/apache/cassandra/db/marshal/CollectionType.java b/src/java/org/apache/cassandra/db/marshal/CollectionType.java index 1660b2e469..0b00b47295 100644 --- a/src/java/org/apache/cassandra/db/marshal/CollectionType.java +++ b/src/java/org/apache/cassandra/db/marshal/CollectionType.java @@ -18,22 +18,28 @@ package org.apache.cassandra.db.marshal; import java.nio.ByteBuffer; +import java.io.DataInput; +import java.io.IOException; import java.util.List; +import java.util.Iterator; -import org.apache.cassandra.config.ColumnDefinition; -import org.apache.cassandra.db.Cell; -import org.apache.cassandra.transport.Server; import org.slf4j.Logger; import org.slf4j.LoggerFactory; +import org.apache.cassandra.config.ColumnDefinition; import org.apache.cassandra.cql3.CQL3Type; import org.apache.cassandra.cql3.ColumnSpecification; import org.apache.cassandra.cql3.Lists; import org.apache.cassandra.cql3.Maps; import org.apache.cassandra.cql3.Sets; - +import org.apache.cassandra.db.TypeSizes; +import org.apache.cassandra.db.rows.Cell; +import org.apache.cassandra.db.rows.CellPath; +import org.apache.cassandra.io.util.DataOutputPlus; +import org.apache.cassandra.io.util.FileUtils; import org.apache.cassandra.serializers.CollectionSerializer; import org.apache.cassandra.serializers.MarshalException; +import org.apache.cassandra.transport.Server; import org.apache.cassandra.utils.ByteBufferUtil; /** @@ -47,6 +53,8 @@ public abstract class CollectionType extends AbstractType public static final int MAX_ELEMENTS = 65535; + public static CellPath.Serializer cellPathSerializer = new CollectionPathSerializer(); + public enum Kind { MAP @@ -84,6 +92,8 @@ public abstract class CollectionType extends AbstractType public abstract AbstractType nameComparator(); public abstract AbstractType valueComparator(); + protected abstract List serializedValues(Iterator cells); + @Override public abstract CollectionSerializer getSerializer(); @@ -132,28 +142,33 @@ public abstract class CollectionType extends AbstractType return kind == Kind.MAP; } - public List enforceLimit(ColumnDefinition def, List cells, int version) + // Overrided by maps + protected int collectionSize(List values) + { + return values.size(); + } + + protected int enforceLimit(ColumnDefinition def, List values, int version) { assert isMultiCell(); - if (version >= Server.VERSION_3 || cells.size() <= MAX_ELEMENTS) - return cells; + int size = collectionSize(values); + if (version >= Server.VERSION_3 || size <= MAX_ELEMENTS) + return size; logger.error("Detected collection for table {}.{} with {} elements, more than the {} limit. Only the first {}" + " elements will be returned to the client. Please see " + "http://cassandra.apache.org/doc/cql3/CQL.html#collections for more details.", - def.ksName, def.cfName, cells.size(), MAX_ELEMENTS, MAX_ELEMENTS); - return cells.subList(0, MAX_ELEMENTS); + def.ksName, def.cfName, values.size(), MAX_ELEMENTS, MAX_ELEMENTS); + return MAX_ELEMENTS; } - public abstract List serializedValues(List cells); - - public ByteBuffer serializeForNativeProtocol(ColumnDefinition def, List cells, int version) + public ByteBuffer serializeForNativeProtocol(ColumnDefinition def, Iterator cells, int version) { assert isMultiCell(); - cells = enforceLimit(def, cells, version); List values = serializedValues(cells); - return CollectionSerializer.pack(values, cells.size(), version); + int size = enforceLimit(def, values, version); + return CollectionSerializer.pack(values, size, version); } @Override @@ -217,4 +232,28 @@ public abstract class CollectionType extends AbstractType { return this.toString(false); } + + private static class CollectionPathSerializer implements CellPath.Serializer + { + public void serialize(CellPath path, DataOutputPlus out) throws IOException + { + ByteBufferUtil.writeWithLength(path.get(0), out); + } + + public CellPath deserialize(DataInput in) throws IOException + { + return CellPath.create(ByteBufferUtil.readWithLength(in)); + } + + public long serializedSize(CellPath path, TypeSizes sizes) + { + return sizes.sizeofWithLength(path.get(0)); + } + + public void skip(DataInput in) throws IOException + { + int length = in.readInt(); + FileUtils.skipBytesFully(in, length); + } + } } diff --git a/src/java/org/apache/cassandra/db/marshal/ColumnToCollectionType.java b/src/java/org/apache/cassandra/db/marshal/ColumnToCollectionType.java index 1d2c88c0e2..a81d3f863c 100644 --- a/src/java/org/apache/cassandra/db/marshal/ColumnToCollectionType.java +++ b/src/java/org/apache/cassandra/db/marshal/ColumnToCollectionType.java @@ -31,6 +31,9 @@ import org.apache.cassandra.serializers.BytesSerializer; import org.apache.cassandra.serializers.MarshalException; import org.apache.cassandra.utils.ByteBufferUtil; +/* + * This class is deprecated and only kept for backward compatibility. + */ public class ColumnToCollectionType extends AbstractType { // interning instances diff --git a/src/java/org/apache/cassandra/db/marshal/CompositeType.java b/src/java/org/apache/cassandra/db/marshal/CompositeType.java index f3c041ef22..01eb58ffb9 100644 --- a/src/java/org/apache/cassandra/db/marshal/CompositeType.java +++ b/src/java/org/apache/cassandra/db/marshal/CompositeType.java @@ -177,6 +177,7 @@ public class CompositeType extends AbstractCompositeType // most names will be complete. ByteBuffer[] l = new ByteBuffer[types.size()]; ByteBuffer bb = name.duplicate(); + readStatic(bb); int i = 0; while (bb.remaining() > 0) { @@ -186,6 +187,19 @@ public class CompositeType extends AbstractCompositeType return i == l.length ? l : Arrays.copyOfRange(l, 0, i); } + public static List splitName(ByteBuffer name) + { + List l = new ArrayList<>(); + ByteBuffer bb = name.duplicate(); + readStatic(bb); + while (bb.remaining() > 0) + { + l.add(ByteBufferUtil.readBytesWithShortLength(bb)); + bb.get(); // skip end-of-component + } + return l; + } + // Extract component idx from bb. Return null if there is not enough component. public static ByteBuffer extractComponent(ByteBuffer bb, int idx) { @@ -318,11 +332,20 @@ public class CompositeType extends AbstractCompositeType public static ByteBuffer build(ByteBuffer... buffers) { - int totalLength = 0; + return build(false, buffers); + } + + public static ByteBuffer build(boolean isStatic, ByteBuffer... buffers) + { + int totalLength = isStatic ? 2 : 0; for (ByteBuffer bb : buffers) totalLength += 2 + bb.remaining() + 1; ByteBuffer out = ByteBuffer.allocate(totalLength); + + if (isStatic) + out.putShort((short)STATIC_MARKER); + for (ByteBuffer bb : buffers) { ByteBufferUtil.writeShortLength(out, bb.remaining()); diff --git a/src/java/org/apache/cassandra/db/marshal/CounterColumnType.java b/src/java/org/apache/cassandra/db/marshal/CounterColumnType.java index 4b3ce82e20..687e525779 100644 --- a/src/java/org/apache/cassandra/db/marshal/CounterColumnType.java +++ b/src/java/org/apache/cassandra/db/marshal/CounterColumnType.java @@ -24,6 +24,7 @@ import org.apache.cassandra.cql3.Term; import org.apache.cassandra.db.context.CounterContext; import org.apache.cassandra.serializers.TypeSerializer; import org.apache.cassandra.serializers.CounterSerializer; +import org.apache.cassandra.serializers.MarshalException; import org.apache.cassandra.utils.ByteBufferUtil; public class CounterColumnType extends AbstractType @@ -59,6 +60,12 @@ public class CounterColumnType extends AbstractType return ByteBufferUtil.bytes(value); } + @Override + public void validateCellValue(ByteBuffer cellValue) throws MarshalException + { + CounterContext.instance().validateContext(cellValue); + } + public int compare(ByteBuffer o1, ByteBuffer o2) { return ByteBufferUtil.compareUnsigned(o1, o2); diff --git a/src/java/org/apache/cassandra/db/marshal/DateType.java b/src/java/org/apache/cassandra/db/marshal/DateType.java index 359ce52bad..66da443c31 100644 --- a/src/java/org/apache/cassandra/db/marshal/DateType.java +++ b/src/java/org/apache/cassandra/db/marshal/DateType.java @@ -124,4 +124,10 @@ public class DateType extends AbstractType { return TimestampSerializer.instance; } + + @Override + protected int valueLengthIfFixed() + { + return 8; + } } diff --git a/src/java/org/apache/cassandra/db/marshal/DoubleType.java b/src/java/org/apache/cassandra/db/marshal/DoubleType.java index 661b3c9809..bc160d5d47 100644 --- a/src/java/org/apache/cassandra/db/marshal/DoubleType.java +++ b/src/java/org/apache/cassandra/db/marshal/DoubleType.java @@ -97,4 +97,10 @@ public class DoubleType extends AbstractType { return DoubleSerializer.instance; } + + @Override + protected int valueLengthIfFixed() + { + return 8; + } } diff --git a/src/java/org/apache/cassandra/db/marshal/EmptyType.java b/src/java/org/apache/cassandra/db/marshal/EmptyType.java index f82d7671bb..448376f471 100644 --- a/src/java/org/apache/cassandra/db/marshal/EmptyType.java +++ b/src/java/org/apache/cassandra/db/marshal/EmptyType.java @@ -69,4 +69,10 @@ public class EmptyType extends AbstractType { return EmptySerializer.instance; } + + @Override + protected int valueLengthIfFixed() + { + return 0; + } } diff --git a/src/java/org/apache/cassandra/db/marshal/FloatType.java b/src/java/org/apache/cassandra/db/marshal/FloatType.java index af02cad56c..ceedce4e29 100644 --- a/src/java/org/apache/cassandra/db/marshal/FloatType.java +++ b/src/java/org/apache/cassandra/db/marshal/FloatType.java @@ -96,4 +96,10 @@ public class FloatType extends AbstractType { return FloatSerializer.instance; } + + @Override + protected int valueLengthIfFixed() + { + return 4; + } } diff --git a/src/java/org/apache/cassandra/db/marshal/Int32Type.java b/src/java/org/apache/cassandra/db/marshal/Int32Type.java index 67d8142e44..cb0c5848fa 100644 --- a/src/java/org/apache/cassandra/db/marshal/Int32Type.java +++ b/src/java/org/apache/cassandra/db/marshal/Int32Type.java @@ -109,4 +109,9 @@ public class Int32Type extends AbstractType return Int32Serializer.instance; } + @Override + protected int valueLengthIfFixed() + { + return 4; + } } diff --git a/src/java/org/apache/cassandra/db/marshal/LexicalUUIDType.java b/src/java/org/apache/cassandra/db/marshal/LexicalUUIDType.java index 3e00d71e8c..174ce3a566 100644 --- a/src/java/org/apache/cassandra/db/marshal/LexicalUUIDType.java +++ b/src/java/org/apache/cassandra/db/marshal/LexicalUUIDType.java @@ -83,4 +83,10 @@ public class LexicalUUIDType extends AbstractType { return UUIDSerializer.instance; } + + @Override + protected int valueLengthIfFixed() + { + return 16; + } } diff --git a/src/java/org/apache/cassandra/db/marshal/ListType.java b/src/java/org/apache/cassandra/db/marshal/ListType.java index 03f39d7fba..73af808754 100644 --- a/src/java/org/apache/cassandra/db/marshal/ListType.java +++ b/src/java/org/apache/cassandra/db/marshal/ListType.java @@ -23,14 +23,13 @@ import java.util.*; import org.apache.cassandra.cql3.Json; import org.apache.cassandra.cql3.Lists; import org.apache.cassandra.cql3.Term; -import org.apache.cassandra.db.Cell; +import org.apache.cassandra.db.rows.Cell; import org.apache.cassandra.exceptions.ConfigurationException; import org.apache.cassandra.exceptions.SyntaxException; import org.apache.cassandra.serializers.CollectionSerializer; import org.apache.cassandra.serializers.MarshalException; import org.apache.cassandra.serializers.ListSerializer; -import org.apache.cassandra.transport.Server; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -169,12 +168,12 @@ public class ListType extends CollectionType> return sb.toString(); } - public List serializedValues(List cells) + public List serializedValues(Iterator cells) { assert isMultiCell; - List bbs = new ArrayList(cells.size()); - for (Cell c : cells) - bbs.add(c.value()); + List bbs = new ArrayList(); + while (cells.hasNext()) + bbs.add(cells.next().value()); return bbs; } diff --git a/src/java/org/apache/cassandra/db/marshal/LocalByPartionerType.java b/src/java/org/apache/cassandra/db/marshal/LocalByPartionerType.java index 427598d0bd..e02ba3c0b6 100644 --- a/src/java/org/apache/cassandra/db/marshal/LocalByPartionerType.java +++ b/src/java/org/apache/cassandra/db/marshal/LocalByPartionerType.java @@ -20,11 +20,12 @@ package org.apache.cassandra.db.marshal; import java.nio.ByteBuffer; import org.apache.cassandra.cql3.Term; +import org.apache.cassandra.db.PartitionPosition; import org.apache.cassandra.serializers.TypeSerializer; import org.apache.cassandra.serializers.MarshalException; -import org.apache.cassandra.db.RowPosition; import org.apache.cassandra.dht.IPartitioner; +import org.apache.cassandra.service.StorageService; import org.apache.cassandra.utils.ByteBufferUtil; /** for sorting columns representing row keys in the row ordering as determined by a partitioner. @@ -38,6 +39,11 @@ public class LocalByPartionerType extends AbstractType this.partitioner = partitioner; } + public static LocalByPartionerType getInstance(TypeParser parser) + { + return new LocalByPartionerType(StorageService.getPartitioner()); + } + @Override public ByteBuffer compose(ByteBuffer bytes) { @@ -74,8 +80,8 @@ public class LocalByPartionerType extends AbstractType public int compare(ByteBuffer o1, ByteBuffer o2) { - // o1 and o2 can be empty so we need to use RowPosition, not DecoratedKey - return RowPosition.ForKey.get(o1, partitioner).compareTo(RowPosition.ForKey.get(o2, partitioner)); + // o1 and o2 can be empty so we need to use PartitionPosition, not DecoratedKey + return PartitionPosition.ForKey.get(o1, partitioner).compareTo(PartitionPosition.ForKey.get(o2, partitioner)); } @Override diff --git a/src/java/org/apache/cassandra/db/marshal/LongType.java b/src/java/org/apache/cassandra/db/marshal/LongType.java index d77d7d0b1d..9d41f4f583 100644 --- a/src/java/org/apache/cassandra/db/marshal/LongType.java +++ b/src/java/org/apache/cassandra/db/marshal/LongType.java @@ -118,4 +118,9 @@ public class LongType extends AbstractType return LongSerializer.instance; } + @Override + protected int valueLengthIfFixed() + { + return 8; + } } diff --git a/src/java/org/apache/cassandra/db/marshal/MapType.java b/src/java/org/apache/cassandra/db/marshal/MapType.java index 983710bf93..28a4fd5043 100644 --- a/src/java/org/apache/cassandra/db/marshal/MapType.java +++ b/src/java/org/apache/cassandra/db/marshal/MapType.java @@ -23,7 +23,7 @@ import java.util.*; import org.apache.cassandra.cql3.Json; import org.apache.cassandra.cql3.Maps; import org.apache.cassandra.cql3.Term; -import org.apache.cassandra.db.Cell; +import org.apache.cassandra.db.rows.Cell; import org.apache.cassandra.exceptions.ConfigurationException; import org.apache.cassandra.exceptions.SyntaxException; import org.apache.cassandra.serializers.CollectionSerializer; @@ -173,6 +173,11 @@ public class MapType extends CollectionType> } @Override + protected int collectionSize(List values) + { + return values.size() / 2; + } + public String toString(boolean ignoreFreezing) { boolean includeFrozenType = !ignoreFreezing && !isMultiCell(); @@ -186,13 +191,14 @@ public class MapType extends CollectionType> return sb.toString(); } - public List serializedValues(List cells) + public List serializedValues(Iterator cells) { assert isMultiCell; - List bbs = new ArrayList(cells.size() * 2); - for (Cell c : cells) + List bbs = new ArrayList(); + while (cells.hasNext()) { - bbs.add(c.name().collectionElement()); + Cell c = cells.next(); + bbs.add(c.path().get(0)); bbs.add(c.value()); } return bbs; diff --git a/src/java/org/apache/cassandra/db/marshal/ReversedType.java b/src/java/org/apache/cassandra/db/marshal/ReversedType.java index 2181f74002..cf357a8772 100644 --- a/src/java/org/apache/cassandra/db/marshal/ReversedType.java +++ b/src/java/org/apache/cassandra/db/marshal/ReversedType.java @@ -80,6 +80,12 @@ public class ReversedType extends AbstractType return baseType.compare(o2, o1); } + @Override + public int compareForCQL(ByteBuffer v1, ByteBuffer v2) + { + return baseType.compare(v1, v2); + } + public String getString(ByteBuffer bytes) { return baseType.getString(bytes); @@ -128,6 +134,12 @@ public class ReversedType extends AbstractType return baseType.getSerializer(); } + @Override + protected int valueLengthIfFixed() + { + return baseType.valueLengthIfFixed(); + } + @Override public String toString() { diff --git a/src/java/org/apache/cassandra/db/marshal/SetType.java b/src/java/org/apache/cassandra/db/marshal/SetType.java index 78aac25aa0..126c6aa228 100644 --- a/src/java/org/apache/cassandra/db/marshal/SetType.java +++ b/src/java/org/apache/cassandra/db/marshal/SetType.java @@ -23,12 +23,11 @@ import java.util.*; import org.apache.cassandra.cql3.Json; import org.apache.cassandra.cql3.Sets; import org.apache.cassandra.cql3.Term; -import org.apache.cassandra.db.Cell; +import org.apache.cassandra.db.rows.Cell; import org.apache.cassandra.exceptions.ConfigurationException; import org.apache.cassandra.exceptions.SyntaxException; import org.apache.cassandra.serializers.MarshalException; import org.apache.cassandra.serializers.SetSerializer; -import org.apache.cassandra.transport.Server; public class SetType extends CollectionType> { @@ -144,11 +143,11 @@ public class SetType extends CollectionType> return sb.toString(); } - public List serializedValues(List cells) + public List serializedValues(Iterator cells) { - List bbs = new ArrayList(cells.size()); - for (Cell c : cells) - bbs.add(c.name().collectionElement()); + List bbs = new ArrayList(); + while (cells.hasNext()) + bbs.add(cells.next().path().get(0)); return bbs; } diff --git a/src/java/org/apache/cassandra/db/marshal/TimeUUIDType.java b/src/java/org/apache/cassandra/db/marshal/TimeUUIDType.java index a1d8d823a5..64fa750281 100644 --- a/src/java/org/apache/cassandra/db/marshal/TimeUUIDType.java +++ b/src/java/org/apache/cassandra/db/marshal/TimeUUIDType.java @@ -127,4 +127,10 @@ public class TimeUUIDType extends AbstractType { return TimeUUIDSerializer.instance; } + + @Override + protected int valueLengthIfFixed() + { + return 16; + } } diff --git a/src/java/org/apache/cassandra/db/marshal/TimestampType.java b/src/java/org/apache/cassandra/db/marshal/TimestampType.java index b01651dbdb..288f8fda2c 100644 --- a/src/java/org/apache/cassandra/db/marshal/TimestampType.java +++ b/src/java/org/apache/cassandra/db/marshal/TimestampType.java @@ -125,4 +125,10 @@ public class TimestampType extends AbstractType { return TimestampSerializer.instance; } + + @Override + protected int valueLengthIfFixed() + { + return 8; + } } diff --git a/src/java/org/apache/cassandra/db/marshal/UUIDType.java b/src/java/org/apache/cassandra/db/marshal/UUIDType.java index 0250eb2002..14c9f48390 100644 --- a/src/java/org/apache/cassandra/db/marshal/UUIDType.java +++ b/src/java/org/apache/cassandra/db/marshal/UUIDType.java @@ -168,4 +168,10 @@ public class UUIDType extends AbstractType { return (uuid.get(6) & 0xf0) >> 4; } + + @Override + protected int valueLengthIfFixed() + { + return 16; + } } diff --git a/src/java/org/apache/cassandra/db/partitions/AbstractPartitionData.java b/src/java/org/apache/cassandra/db/partitions/AbstractPartitionData.java new file mode 100644 index 0000000000..2fcd7b3a85 --- /dev/null +++ b/src/java/org/apache/cassandra/db/partitions/AbstractPartitionData.java @@ -0,0 +1,831 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.cassandra.db.partitions; + +import java.nio.ByteBuffer; +import java.util.*; + +import com.google.common.collect.AbstractIterator; +import com.google.common.collect.UnmodifiableIterator; + +import org.apache.cassandra.config.CFMetaData; +import org.apache.cassandra.config.ColumnDefinition; +import org.apache.cassandra.db.*; +import org.apache.cassandra.db.filter.ColumnFilter; +import org.apache.cassandra.db.rows.*; +import org.apache.cassandra.utils.SearchIterator; + +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +/** + * Abstract common class for all non-thread safe Partition implementations. + */ +public abstract class AbstractPartitionData implements Partition, Iterable +{ + private static final Logger logger = LoggerFactory.getLogger(AbstractPartitionData.class); + + protected final CFMetaData metadata; + protected final DecoratedKey key; + + protected final DeletionInfo deletionInfo; + protected final PartitionColumns columns; + + protected Row staticRow; + + protected int rows; + + // The values for the clustering columns of the rows contained in this partition object. If + // clusteringSize is the size of the clustering comparator for this table, clusterings has size + // clusteringSize * rows where rows is the number of rows stored, and row i has it's clustering + // column values at indexes [clusteringSize * i, clusteringSize * (i + 1)). + protected ByteBuffer[] clusterings; + + // The partition key column liveness infos for the rows of this partition (row i has its liveness info at index i). + protected final LivenessInfoArray livenessInfos; + // The row deletion for the rows of this partition (row i has its row deletion at index i). + protected final DeletionTimeArray deletions; + + // The row data (cells data + complex deletions for complex columns) for the rows contained in this partition. + protected final RowDataBlock data; + + // Stats over the rows stored in this partition. + private final RowStats.Collector statsCollector = new RowStats.Collector(); + + // The maximum timestamp for any data contained in this partition. + protected long maxTimestamp = Long.MIN_VALUE; + + private AbstractPartitionData(CFMetaData metadata, + DecoratedKey key, + DeletionInfo deletionInfo, + ByteBuffer[] clusterings, + LivenessInfoArray livenessInfos, + DeletionTimeArray deletions, + PartitionColumns columns, + RowDataBlock data) + { + this.metadata = metadata; + this.key = key; + this.deletionInfo = deletionInfo; + this.clusterings = clusterings; + this.livenessInfos = livenessInfos; + this.deletions = deletions; + this.columns = columns; + this.data = data; + + collectStats(deletionInfo.getPartitionDeletion()); + Iterator iter = deletionInfo.rangeIterator(false); + while (iter.hasNext()) + collectStats(iter.next().deletionTime()); + } + + protected AbstractPartitionData(CFMetaData metadata, + DecoratedKey key, + DeletionInfo deletionInfo, + PartitionColumns columns, + RowDataBlock data, + int initialRowCapacity) + { + this(metadata, + key, + deletionInfo, + new ByteBuffer[initialRowCapacity * metadata.clusteringColumns().size()], + new LivenessInfoArray(initialRowCapacity), + new DeletionTimeArray(initialRowCapacity), + columns, + data); + } + + protected AbstractPartitionData(CFMetaData metadata, + DecoratedKey key, + DeletionTime partitionDeletion, + PartitionColumns columns, + int initialRowCapacity, + boolean sortable) + { + this(metadata, + key, + new DeletionInfo(partitionDeletion.takeAlias()), + columns, + new RowDataBlock(columns.regulars, initialRowCapacity, sortable, metadata.isCounter()), + initialRowCapacity); + } + + private void collectStats(DeletionTime dt) + { + statsCollector.updateDeletionTime(dt); + maxTimestamp = Math.max(maxTimestamp, dt.markedForDeleteAt()); + } + + private void collectStats(LivenessInfo info) + { + statsCollector.updateTimestamp(info.timestamp()); + statsCollector.updateTTL(info.ttl()); + statsCollector.updateLocalDeletionTime(info.localDeletionTime()); + maxTimestamp = Math.max(maxTimestamp, info.timestamp()); + } + + public CFMetaData metadata() + { + return metadata; + } + + public DecoratedKey partitionKey() + { + return key; + } + + public DeletionTime partitionLevelDeletion() + { + return deletionInfo.getPartitionDeletion(); + } + + public PartitionColumns columns() + { + return columns; + } + + public Row staticRow() + { + return staticRow == null ? Rows.EMPTY_STATIC_ROW : staticRow; + } + + public RowStats stats() + { + return statsCollector.get(); + } + + /** + * The deletion info for the partition update. + * + * warning: the returned object should be used in a read-only fashion. In particular, + * it should not be used to add new range tombstones to this deletion. For that, + * {@link addRangeTombstone} should be used instead. The reason being that adding directly to + * the returned object would bypass some stats collection that {@code addRangeTombstone} does. + * + * @return the deletion info for the partition update for use as read-only. + */ + public DeletionInfo deletionInfo() + { + // TODO: it is a tad fragile that deletionInfo can be but shouldn't be modified. We + // could add the option of providing a read-only view of a DeletionInfo instead. + return deletionInfo; + } + + public void addPartitionDeletion(DeletionTime deletionTime) + { + collectStats(deletionTime); + deletionInfo.add(deletionTime); + } + + public void addRangeTombstone(Slice deletedSlice, DeletionTime deletion) + { + addRangeTombstone(new RangeTombstone(deletedSlice, deletion.takeAlias())); + } + + public void addRangeTombstone(RangeTombstone range) + { + collectStats(range.deletionTime()); + deletionInfo.add(range, metadata.comparator); + } + + /** + * Swap row i and j. + * + * This is only used when we need to reorder rows because those were not added in clustering order, + * which happens in {@link PartitionUpdate#sort} and {@link ArrayBackedPartition#create}. This method + * is public only because {@code PartitionUpdate} needs to implement {@link Sorting.Sortable}, but + * it should really only be used by subclasses (and with care) in practice. + */ + public void swap(int i, int j) + { + int cs = metadata.clusteringColumns().size(); + for (int k = 0; k < cs; k++) + { + ByteBuffer tmp = clusterings[j * cs + k]; + clusterings[j * cs + k] = clusterings[i * cs + k]; + clusterings[i * cs + k] = tmp; + } + + livenessInfos.swap(i, j); + deletions.swap(i, j); + data.swap(i, j); + } + + public int rowCount() + { + return rows; + } + + public boolean isEmpty() + { + return deletionInfo.isLive() && rows == 0 && staticRow().isEmpty(); + } + + protected void clear() + { + rows = 0; + Arrays.fill(clusterings, null); + livenessInfos.clear(); + deletions.clear(); + data.clear(); + } + + @Override + public String toString() + { + StringBuilder sb = new StringBuilder(); + CFMetaData metadata = metadata(); + sb.append(String.format("Partition[%s.%s] key=%s columns=%s deletion=%s", + metadata.ksName, + metadata.cfName, + metadata.getKeyValidator().getString(partitionKey().getKey()), + columns(), + deletionInfo)); + + if (staticRow() != Rows.EMPTY_STATIC_ROW) + sb.append("\n ").append(staticRow().toString(metadata, true)); + + // We use createRowIterator() directly instead of iterator() because that avoids + // sorting for PartitionUpdate (which inherit this method) and that is useful because + // 1) it can help with debugging and 2) we can't write after sorting but we want to + // be able to print an update while we build it (again for debugging) + Iterator iterator = createRowIterator(null, false); + while (iterator.hasNext()) + sb.append("\n ").append(iterator.next().toString(metadata, true)); + + return sb.toString(); + } + + protected void reverse() + { + for (int i = 0; i < rows / 2; i++) + swap(i, rows - 1 - i); + } + + public Row getRow(Clustering clustering) + { + Row row = searchIterator(ColumnFilter.selection(columns()), false).next(clustering); + // Note that for statics, this will never return null, this will return an empty row. However, + // it's more consistent for this method to return null if we don't really have a static row. + return row == null || (clustering == Clustering.STATIC_CLUSTERING && row.isEmpty()) ? null : row; + } + + /** + * Returns an iterator that iterators over the rows of this update in clustering order. + * + * @return an iterator over the rows of this update. + */ + public Iterator iterator() + { + return createRowIterator(null, false); + } + + public SearchIterator searchIterator(final ColumnFilter columns, boolean reversed) + { + final RowIterator iter = createRowIterator(columns, reversed); + return new SearchIterator() + { + public boolean hasNext() + { + return iter.hasNext(); + } + + public Row next(Clustering key) + { + if (key == Clustering.STATIC_CLUSTERING) + { + if (columns.fetchedColumns().statics.isEmpty() || staticRow().isEmpty()) + return Rows.EMPTY_STATIC_ROW; + + return FilteringRow.columnsFilteringRow(columns).setTo(staticRow()); + } + + return iter.seekTo(key) ? iter.next() : null; + } + }; + } + + public UnfilteredRowIterator unfilteredIterator() + { + return unfilteredIterator(ColumnFilter.selection(columns()), Slices.ALL, false); + } + + public UnfilteredRowIterator unfilteredIterator(ColumnFilter columns, Slices slices, boolean reversed) + { + return slices.makeSliceIterator(sliceableUnfilteredIterator(columns, reversed)); + } + + protected SliceableUnfilteredRowIterator sliceableUnfilteredIterator() + { + return sliceableUnfilteredIterator(ColumnFilter.selection(columns()), false); + } + + protected SliceableUnfilteredRowIterator sliceableUnfilteredIterator(final ColumnFilter selection, final boolean reversed) + { + return new AbstractSliceableIterator(this, selection.fetchedColumns(), reversed) + { + private final RowIterator rowIterator = createRowIterator(selection, reversed); + private RowAndTombstoneMergeIterator mergeIterator = new RowAndTombstoneMergeIterator(metadata.comparator, reversed); + + protected Unfiltered computeNext() + { + if (!mergeIterator.isSet()) + mergeIterator.setTo(rowIterator, deletionInfo.rangeIterator(reversed)); + + return mergeIterator.hasNext() ? mergeIterator.next() : endOfData(); + } + + public Iterator slice(Slice slice) + { + return mergeIterator.setTo(rowIterator.slice(slice), deletionInfo.rangeIterator(slice, reversed)); + } + }; + } + + private RowIterator createRowIterator(ColumnFilter columns, boolean reversed) + { + return reversed ? new ReverseRowIterator(columns) : new ForwardRowIterator(columns); + } + + /** + * An iterator over the rows of this partition that reuse the same row object. + */ + private abstract class RowIterator extends UnmodifiableIterator + { + protected final InternalReusableClustering clustering = new InternalReusableClustering(); + protected final InternalReusableRow reusableRow; + protected final FilteringRow filter; + + protected int next; + + protected RowIterator(final ColumnFilter columns) + { + this.reusableRow = new InternalReusableRow(clustering); + this.filter = columns == null ? null : FilteringRow.columnsFilteringRow(columns); + } + + /* + * Move the iterator so that row {@code name} is returned next by {@code next} if that + * row exists. Otherwise the first row sorting after {@code name} will be returned. + * Returns whether {@code name} was found or not. + */ + public abstract boolean seekTo(Clustering name); + + public abstract Iterator slice(Slice slice); + + protected Row setRowTo(int row) + { + reusableRow.setTo(row); + return filter == null ? reusableRow : filter.setTo(reusableRow); + } + + /** + * Simple binary search. + */ + protected int binarySearch(ClusteringPrefix name, int fromIndex, int toIndex) + { + int low = fromIndex; + int mid = toIndex; + int high = mid - 1; + int result = -1; + while (low <= high) + { + mid = (low + high) >> 1; + if ((result = metadata.comparator.compare(name, clustering.setTo(mid))) > 0) + low = mid + 1; + else if (result == 0) + return mid; + else + high = mid - 1; + } + return -mid - (result < 0 ? 1 : 2); + } + } + + private class ForwardRowIterator extends RowIterator + { + private ForwardRowIterator(ColumnFilter columns) + { + super(columns); + this.next = 0; + } + + public boolean hasNext() + { + return next < rows; + } + + public Row next() + { + return setRowTo(next++); + } + + public boolean seekTo(Clustering name) + { + if (next >= rows) + return false; + + int idx = binarySearch(name, next, rows); + next = idx >= 0 ? idx : -idx - 1; + return idx >= 0; + } + + public Iterator slice(Slice slice) + { + int sidx = binarySearch(slice.start(), next, rows); + final int start = sidx >= 0 ? sidx : -sidx - 1; + if (start >= rows) + return Collections.emptyIterator(); + + int eidx = binarySearch(slice.end(), start, rows); + // The insertion point is the first element greater than slice.end(), so we want the previous index + final int end = eidx >= 0 ? eidx : -eidx - 2; + + // Remember the end to speed up potential further slice search + next = end; + + if (start > end) + return Collections.emptyIterator(); + + return new AbstractIterator() + { + private int i = start; + + protected Row computeNext() + { + if (i >= rows || i > end) + return endOfData(); + + return setRowTo(i++); + } + }; + } + } + + private class ReverseRowIterator extends RowIterator + { + private ReverseRowIterator(ColumnFilter columns) + { + super(columns); + this.next = rows - 1; + } + + public boolean hasNext() + { + return next >= 0; + } + + public Row next() + { + return setRowTo(next--); + } + + public boolean seekTo(Clustering name) + { + // We only use that method with forward iterators. + throw new UnsupportedOperationException(); + } + + public Iterator slice(Slice slice) + { + int sidx = binarySearch(slice.end(), 0, next + 1); + // The insertion point is the first element greater than slice.end(), so we want the previous index + final int start = sidx >= 0 ? sidx : -sidx - 2; + if (start < 0) + return Collections.emptyIterator(); + + int eidx = binarySearch(slice.start(), 0, start + 1); + final int end = eidx >= 0 ? eidx : -eidx - 1; + + // Remember the end to speed up potential further slice search + next = end; + + if (start < end) + return Collections.emptyIterator(); + + return new AbstractIterator() + { + private int i = start; + + protected Row computeNext() + { + if (i < 0 || i < end) + return endOfData(); + + return setRowTo(i--); + } + }; + } + } + + /** + * A reusable view over the clustering of this partition. + */ + protected class InternalReusableClustering extends Clustering + { + final int size = metadata.clusteringColumns().size(); + private int base; + + public int size() + { + return size; + } + + public Clustering setTo(int row) + { + base = row * size; + return this; + } + + public ByteBuffer get(int i) + { + return clusterings[base + i]; + } + + public ByteBuffer[] getRawValues() + { + ByteBuffer[] values = new ByteBuffer[size]; + for (int i = 0; i < size; i++) + values[i] = get(i); + return values; + } + }; + + /** + * A reusable view over the rows of this partition. + */ + protected class InternalReusableRow extends AbstractReusableRow + { + private final LivenessInfoArray.Cursor liveness = new LivenessInfoArray.Cursor(); + private final DeletionTimeArray.Cursor deletion = new DeletionTimeArray.Cursor(); + private final InternalReusableClustering clustering; + + private int row; + + public InternalReusableRow() + { + this(new InternalReusableClustering()); + } + + public InternalReusableRow(InternalReusableClustering clustering) + { + this.clustering = clustering; + } + + protected RowDataBlock data() + { + return data; + } + + public Row setTo(int row) + { + this.clustering.setTo(row); + this.liveness.setTo(livenessInfos, row); + this.deletion.setTo(deletions, row); + this.row = row; + return this; + } + + protected int row() + { + return row; + } + + public Clustering clustering() + { + return clustering; + } + + public LivenessInfo primaryKeyLivenessInfo() + { + return liveness; + } + + public DeletionTime deletion() + { + return deletion; + } + }; + + private static abstract class AbstractSliceableIterator extends AbstractUnfilteredRowIterator implements SliceableUnfilteredRowIterator + { + private AbstractSliceableIterator(AbstractPartitionData data, PartitionColumns columns, boolean isReverseOrder) + { + super(data.metadata, data.key, data.partitionLevelDeletion(), columns, data.staticRow(), isReverseOrder, data.stats()); + } + } + + /** + * A row writer to add rows to this partition. + */ + protected class Writer extends RowDataBlock.Writer + { + private int clusteringBase; + + private int simpleColumnsSetInRow; + private final Set complexColumnsSetInRow = new HashSet<>(); + + public Writer(boolean inOrderCells) + { + super(data, inOrderCells); + } + + public void writeClusteringValue(ByteBuffer value) + { + ensureCapacity(row); + clusterings[clusteringBase++] = value; + } + + public void writePartitionKeyLivenessInfo(LivenessInfo info) + { + ensureCapacity(row); + livenessInfos.set(row, info); + collectStats(info); + } + + public void writeRowDeletion(DeletionTime deletion) + { + ensureCapacity(row); + if (!deletion.isLive()) + deletions.set(row, deletion); + + collectStats(deletion); + } + + @Override + public void writeCell(ColumnDefinition column, boolean isCounter, ByteBuffer value, LivenessInfo info, CellPath path) + { + ensureCapacity(row); + collectStats(info); + + if (column.isComplex()) + complexColumnsSetInRow.add(column); + else + ++simpleColumnsSetInRow; + + super.writeCell(column, isCounter, value, info, path); + } + + @Override + public void writeComplexDeletion(ColumnDefinition c, DeletionTime complexDeletion) + { + ensureCapacity(row); + collectStats(complexDeletion); + + super.writeComplexDeletion(c, complexDeletion); + } + + @Override + public void endOfRow() + { + super.endOfRow(); + ++rows; + + statsCollector.updateColumnSetPerRow(simpleColumnsSetInRow + complexColumnsSetInRow.size()); + + simpleColumnsSetInRow = 0; + complexColumnsSetInRow.clear(); + } + + public int currentRow() + { + return row; + } + + private void ensureCapacity(int rowToSet) + { + int originalCapacity = livenessInfos.size(); + if (rowToSet < originalCapacity) + return; + + int newCapacity = RowDataBlock.computeNewCapacity(originalCapacity, rowToSet); + + int clusteringSize = metadata.clusteringColumns().size(); + + clusterings = Arrays.copyOf(clusterings, newCapacity * clusteringSize); + + livenessInfos.resize(newCapacity); + deletions.resize(newCapacity); + } + + @Override + public Writer reset() + { + super.reset(); + clusteringBase = 0; + simpleColumnsSetInRow = 0; + complexColumnsSetInRow.clear(); + return this; + } + } + + /** + * A range tombstone marker writer to add range tombstone markers to this partition. + */ + protected class RangeTombstoneCollector implements RangeTombstoneMarker.Writer + { + private final boolean reversed; + + private final ByteBuffer[] nextValues = new ByteBuffer[metadata().comparator.size()]; + private int size; + private RangeTombstone.Bound.Kind nextKind; + + private Slice.Bound openBound; + private DeletionTime openDeletion; + + public RangeTombstoneCollector(boolean reversed) + { + this.reversed = reversed; + } + + public void writeClusteringValue(ByteBuffer value) + { + nextValues[size++] = value; + } + + public void writeBoundKind(RangeTombstone.Bound.Kind kind) + { + nextKind = kind; + } + + private ByteBuffer[] getValues() + { + return Arrays.copyOfRange(nextValues, 0, size); + } + + private void open(RangeTombstone.Bound.Kind kind, DeletionTime deletion) + { + openBound = Slice.Bound.create(kind, getValues()); + openDeletion = deletion.takeAlias(); + } + + private void close(RangeTombstone.Bound.Kind kind, DeletionTime deletion) + { + assert deletion.equals(openDeletion) : "Expected " + openDeletion + " but was " + deletion; + Slice.Bound closeBound = Slice.Bound.create(kind, getValues()); + Slice slice = reversed + ? Slice.make(closeBound, openBound) + : Slice.make(openBound, closeBound); + addRangeTombstone(slice, openDeletion); + } + + public void writeBoundDeletion(DeletionTime deletion) + { + assert !nextKind.isBoundary(); + if (nextKind.isOpen(reversed)) + open(nextKind, deletion); + else + close(nextKind, deletion); + } + + public void writeBoundaryDeletion(DeletionTime endDeletion, DeletionTime startDeletion) + { + assert nextKind.isBoundary(); + DeletionTime closeTime = reversed ? startDeletion : endDeletion; + DeletionTime openTime = reversed ? endDeletion : startDeletion; + + close(nextKind.closeBoundOfBoundary(reversed), closeTime); + open(nextKind.openBoundOfBoundary(reversed), openTime); + } + + public void endOfMarker() + { + clear(); + } + + private void addRangeTombstone(Slice deletionSlice, DeletionTime dt) + { + AbstractPartitionData.this.addRangeTombstone(deletionSlice, dt); + } + + private void clear() + { + size = 0; + Arrays.fill(nextValues, null); + nextKind = null; + } + + public void reset() + { + openBound = null; + openDeletion = null; + clear(); + } + } +} diff --git a/src/java/org/apache/cassandra/db/CounterUpdateCell.java b/src/java/org/apache/cassandra/db/partitions/AbstractUnfilteredPartitionIterator.java similarity index 65% rename from src/java/org/apache/cassandra/db/CounterUpdateCell.java rename to src/java/org/apache/cassandra/db/partitions/AbstractUnfilteredPartitionIterator.java index 58ac3659d4..d615ea9e11 100644 --- a/src/java/org/apache/cassandra/db/CounterUpdateCell.java +++ b/src/java/org/apache/cassandra/db/partitions/AbstractUnfilteredPartitionIterator.java @@ -15,16 +15,16 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -package org.apache.cassandra.db; +package org.apache.cassandra.db.partitions; -/** - * A counter update while it hasn't been applied yet by the leader replica. - * - * Contains a single counter update. When applied by the leader replica, this - * is transformed to a relevant CounterCell. This Cell is a temporary data - * structure that should never be stored inside a memtable or an sstable. - */ -public interface CounterUpdateCell extends Cell +public abstract class AbstractUnfilteredPartitionIterator implements UnfilteredPartitionIterator { - public long delta(); + public void remove() + { + throw new UnsupportedOperationException(); + } + + public void close() + { + } } diff --git a/src/java/org/apache/cassandra/db/partitions/ArrayBackedCachedPartition.java b/src/java/org/apache/cassandra/db/partitions/ArrayBackedCachedPartition.java new file mode 100644 index 0000000000..68b397086d --- /dev/null +++ b/src/java/org/apache/cassandra/db/partitions/ArrayBackedCachedPartition.java @@ -0,0 +1,256 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.cassandra.db.partitions; + +import java.io.DataInput; +import java.io.IOException; +import java.nio.ByteBuffer; + +import org.apache.cassandra.config.CFMetaData; +import org.apache.cassandra.config.ColumnDefinition; +import org.apache.cassandra.db.*; +import org.apache.cassandra.db.rows.*; +import org.apache.cassandra.io.ISerializer; +import org.apache.cassandra.io.util.DataOutputPlus; +import org.apache.cassandra.net.MessagingService; + +public class ArrayBackedCachedPartition extends ArrayBackedPartition implements CachedPartition +{ + private final int createdAtInSec; + + // Note that those fields are really immutable, but we can't easily pass their values to + // the ctor so they are not final. + private int cachedLiveRows; + private int rowsWithNonExpiringCells; + + private int nonTombstoneCellCount; + private int nonExpiringLiveCells; + + private ArrayBackedCachedPartition(CFMetaData metadata, + DecoratedKey partitionKey, + DeletionTime deletionTime, + PartitionColumns columns, + int initialRowCapacity, + boolean sortable, + int createdAtInSec) + { + super(metadata, partitionKey, deletionTime, columns, initialRowCapacity, sortable); + this.createdAtInSec = createdAtInSec; + } + + /** + * Creates an {@code ArrayBackedCachedPartition} holding all the data of the provided iterator. + * + * Warning: Note that this method does not close the provided iterator and it is + * up to the caller to do so. + * + * @param iterator the iterator got gather in memory. + * @param nowInSec the time of the creation in seconds. This is the time at which {@link #cachedLiveRows} applies. + * @return the created partition. + */ + public static ArrayBackedCachedPartition create(UnfilteredRowIterator iterator, int nowInSec) + { + return create(iterator, 4, nowInSec); + } + + /** + * Creates an {@code ArrayBackedCachedPartition} holding all the data of the provided iterator. + * + * Warning: Note that this method does not close the provided iterator and it is + * up to the caller to do so. + * + * @param iterator the iterator got gather in memory. + * @param initialRowCapacity sizing hint (in rows) to use for the created partition. It should ideally + * correspond or be a good estimation of the number or rows in {@code iterator}. + * @param nowInSec the time of the creation in seconds. This is the time at which {@link #cachedLiveRows} applies. + * @return the created partition. + */ + public static ArrayBackedCachedPartition create(UnfilteredRowIterator iterator, int initialRowCapacity, int nowInSec) + { + ArrayBackedCachedPartition partition = new ArrayBackedCachedPartition(iterator.metadata(), + iterator.partitionKey(), + iterator.partitionLevelDeletion(), + iterator.columns(), + initialRowCapacity, + iterator.isReverseOrder(), + nowInSec); + + partition.staticRow = iterator.staticRow().takeAlias(); + + Writer writer = partition.new Writer(nowInSec); + RangeTombstoneCollector markerCollector = partition.new RangeTombstoneCollector(iterator.isReverseOrder()); + + copyAll(iterator, writer, markerCollector, partition); + + return partition; + } + + public Row lastRow() + { + if (rows == 0) + return null; + + return new InternalReusableRow().setTo(rows - 1); + } + + /** + * The number of rows that were live at the time the partition was cached. + * + * See {@link ColumnFamilyStore#isFilterFullyCoveredBy} to see why we need this. + * + * @return the number of rows in this partition that were live at the time the + * partition was cached (this can be different from the number of live rows now + * due to expiring cells). + */ + public int cachedLiveRows() + { + return cachedLiveRows; + } + + /** + * The number of rows in this cached partition that have at least one non-expiring + * non-deleted cell. + * + * Note that this is generally not a very meaningful number, but this is used by + * {@link DataLimits#hasEnoughLiveData} as an optimization. + * + * @return the number of row that have at least one non-expiring non-deleted cell. + */ + public int rowsWithNonExpiringCells() + { + return rowsWithNonExpiringCells; + } + + public int nonTombstoneCellCount() + { + return nonTombstoneCellCount; + } + + public int nonExpiringLiveCells() + { + return nonExpiringLiveCells; + } + + // Writers that collect the values for 'cachedLiveRows', 'rowsWithNonExpiringCells', 'nonTombstoneCellCount' + // and 'nonExpiringLiveCells'. + protected class Writer extends AbstractPartitionData.Writer + { + private final int nowInSec; + + private boolean hasLiveData; + private boolean hasNonExpiringCell; + + protected Writer(int nowInSec) + { + super(true); + this.nowInSec = nowInSec; + } + + @Override + public void writePartitionKeyLivenessInfo(LivenessInfo info) + { + super.writePartitionKeyLivenessInfo(info); + if (info.isLive(nowInSec)) + hasLiveData = true; + } + + @Override + public void writeCell(ColumnDefinition column, boolean isCounter, ByteBuffer value, LivenessInfo info, CellPath path) + { + super.writeCell(column, isCounter, value, info, path); + + if (info.isLive(nowInSec)) + { + hasLiveData = true; + if (!info.hasTTL()) + { + hasNonExpiringCell = true; + ++ArrayBackedCachedPartition.this.nonExpiringLiveCells; + } + } + + if (!info.hasLocalDeletionTime() || info.hasTTL()) + ++ArrayBackedCachedPartition.this.nonTombstoneCellCount; + } + + @Override + public void endOfRow() + { + super.endOfRow(); + if (hasLiveData) + ++ArrayBackedCachedPartition.this.cachedLiveRows; + if (hasNonExpiringCell) + ++ArrayBackedCachedPartition.this.rowsWithNonExpiringCells; + + hasLiveData = false; + hasNonExpiringCell = false; + } + } + + static class Serializer implements ISerializer + { + public void serialize(CachedPartition partition, DataOutputPlus out) throws IOException + { + assert partition instanceof ArrayBackedCachedPartition; + ArrayBackedCachedPartition p = (ArrayBackedCachedPartition)partition; + + out.writeInt(p.createdAtInSec); + try (UnfilteredRowIterator iter = p.sliceableUnfilteredIterator()) + { + UnfilteredRowIteratorSerializer.serializer.serialize(iter, out, MessagingService.current_version, p.rows); + } + } + + public CachedPartition deserialize(DataInput in) throws IOException + { + // Note that it would be slightly simpler to just do + // ArrayBackedCachedPiartition.create(UnfilteredRowIteratorSerializer.serializer.deserialize(...)); + // However deserializing the header separatly is not a lot harder and allows us to: + // 1) get the capacity of the partition so we can size it properly directly + // 2) saves the creation of a temporary iterator: rows are directly written to the partition, which + // is slightly faster. + + int createdAtInSec = in.readInt(); + + UnfilteredRowIteratorSerializer.Header h = UnfilteredRowIteratorSerializer.serializer.deserializeHeader(in, MessagingService.current_version, SerializationHelper.Flag.LOCAL); + assert !h.isReversed && h.rowEstimate >= 0; + + ArrayBackedCachedPartition partition = new ArrayBackedCachedPartition(h.metadata, h.key, h.partitionDeletion, h.sHeader.columns(), h.rowEstimate, false, createdAtInSec); + partition.staticRow = h.staticRow; + + Writer writer = partition.new Writer(createdAtInSec); + RangeTombstoneMarker.Writer markerWriter = partition.new RangeTombstoneCollector(false); + + UnfilteredRowIteratorSerializer.serializer.deserialize(in, new SerializationHelper(MessagingService.current_version, SerializationHelper.Flag.LOCAL), h.sHeader, writer, markerWriter); + return partition; + } + + public long serializedSize(CachedPartition partition, TypeSizes sizes) + { + assert partition instanceof ArrayBackedCachedPartition; + ArrayBackedCachedPartition p = (ArrayBackedCachedPartition)partition; + + try (UnfilteredRowIterator iter = p.sliceableUnfilteredIterator()) + { + return sizes.sizeof(p.createdAtInSec) + + UnfilteredRowIteratorSerializer.serializer.serializedSize(iter, MessagingService.current_version, p.rows, sizes); + } + } + } +} + diff --git a/src/java/org/apache/cassandra/db/partitions/ArrayBackedPartition.java b/src/java/org/apache/cassandra/db/partitions/ArrayBackedPartition.java new file mode 100644 index 0000000000..d7f3a884b6 --- /dev/null +++ b/src/java/org/apache/cassandra/db/partitions/ArrayBackedPartition.java @@ -0,0 +1,104 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.cassandra.db.partitions; + +import java.io.DataInput; +import java.io.IOException; +import java.nio.ByteBuffer; + +import org.apache.cassandra.config.CFMetaData; +import org.apache.cassandra.config.ColumnDefinition; +import org.apache.cassandra.db.*; +import org.apache.cassandra.db.rows.*; +import org.apache.cassandra.io.ISerializer; +import org.apache.cassandra.io.util.DataOutputPlus; +import org.apache.cassandra.net.MessagingService; + +public class ArrayBackedPartition extends AbstractPartitionData +{ + protected ArrayBackedPartition(CFMetaData metadata, + DecoratedKey partitionKey, + DeletionTime deletionTime, + PartitionColumns columns, + int initialRowCapacity, + boolean sortable) + { + super(metadata, partitionKey, deletionTime, columns, initialRowCapacity, sortable); + } + + /** + * Creates an {@code ArrayBackedPartition} holding all the data of the provided iterator. + * + * Warning: Note that this method does not close the provided iterator and it is + * up to the caller to do so. + * + * @param iterator the iterator to gather in memory. + * @return the created partition. + */ + public static ArrayBackedPartition create(UnfilteredRowIterator iterator) + { + return create(iterator, 4); + } + + /** + * Creates an {@code ArrayBackedPartition} holding all the data of the provided iterator. + * + * Warning: Note that this method does not close the provided iterator and it is + * up to the caller to do so. + * + * @param iterator the iterator to gather in memory. + * @param initialRowCapacity sizing hint (in rows) to use for the created partition. It should ideally + * correspond or be a good estimation of the number or rows in {@code iterator}. + * @return the created partition. + */ + public static ArrayBackedPartition create(UnfilteredRowIterator iterator, int initialRowCapacity) + { + ArrayBackedPartition partition = new ArrayBackedPartition(iterator.metadata(), + iterator.partitionKey(), + iterator.partitionLevelDeletion(), + iterator.columns(), + initialRowCapacity, + iterator.isReverseOrder()); + + partition.staticRow = iterator.staticRow().takeAlias(); + + Writer writer = partition.new Writer(true); + RangeTombstoneCollector markerCollector = partition.new RangeTombstoneCollector(iterator.isReverseOrder()); + + copyAll(iterator, writer, markerCollector, partition); + + return partition; + } + + protected static void copyAll(UnfilteredRowIterator iterator, Writer writer, RangeTombstoneCollector markerCollector, ArrayBackedPartition partition) + { + while (iterator.hasNext()) + { + Unfiltered unfiltered = iterator.next(); + if (unfiltered.kind() == Unfiltered.Kind.ROW) + ((Row) unfiltered).copyTo(writer); + else + ((RangeTombstoneMarker) unfiltered).copyTo(markerCollector); + } + + // A Partition (or more precisely AbstractPartitionData) always assumes that its data is in clustering + // order. So if we've just added them in reverse clustering order, reverse them. + if (iterator.isReverseOrder()) + partition.reverse(); + } +} diff --git a/src/java/org/apache/cassandra/db/partitions/AtomicBTreePartition.java b/src/java/org/apache/cassandra/db/partitions/AtomicBTreePartition.java new file mode 100644 index 0000000000..4edd7079b4 --- /dev/null +++ b/src/java/org/apache/cassandra/db/partitions/AtomicBTreePartition.java @@ -0,0 +1,819 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.cassandra.db.partitions; + +import java.nio.ByteBuffer; +import java.util.ArrayList; +import java.util.Iterator; +import java.util.List; +import java.util.concurrent.atomic.AtomicIntegerFieldUpdater; +import java.util.concurrent.atomic.AtomicReferenceFieldUpdater; + +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; +import com.google.common.collect.AbstractIterator; +import com.google.common.collect.Iterators; + +import org.apache.cassandra.config.CFMetaData; +import org.apache.cassandra.config.ColumnDefinition; +import org.apache.cassandra.db.*; +import org.apache.cassandra.db.rows.*; +import org.apache.cassandra.db.filter.ColumnFilter; +import org.apache.cassandra.db.index.SecondaryIndexManager; +import org.apache.cassandra.utils.btree.BTree; +import org.apache.cassandra.utils.btree.BTreeSearchIterator; +import org.apache.cassandra.utils.btree.UpdateFunction; +import org.apache.cassandra.utils.ObjectSizes; +import org.apache.cassandra.utils.FBUtilities; +import org.apache.cassandra.utils.SearchIterator; +import org.apache.cassandra.utils.concurrent.OpOrder; +import org.apache.cassandra.utils.concurrent.Locks; +import org.apache.cassandra.utils.memory.MemtableAllocator; +import org.apache.cassandra.utils.memory.HeapAllocator; +import org.apache.cassandra.service.StorageService; +import static org.apache.cassandra.db.index.SecondaryIndexManager.Updater; + +/** + * A thread-safe and atomic Partition implementation. + * + * Operations (in particular addAll) on this implementation are atomic and + * isolated (in the sense of ACID). Typically a addAll is guaranteed that no + * other thread can see the state where only parts but not all rows have + * been added. + */ +public class AtomicBTreePartition implements Partition +{ + private static final Logger logger = LoggerFactory.getLogger(AtomicBTreePartition.class); + + public static final long EMPTY_SIZE = ObjectSizes.measure(new AtomicBTreePartition(CFMetaData.createFake("keyspace", "table"), + StorageService.getPartitioner().decorateKey(ByteBuffer.allocate(1)), + null)); + + // Reserved values for wasteTracker field. These values must not be consecutive (see avoidReservedValues) + private static final int TRACKER_NEVER_WASTED = 0; + private static final int TRACKER_PESSIMISTIC_LOCKING = Integer.MAX_VALUE; + + // The granularity with which we track wasted allocation/work; we round up + private static final int ALLOCATION_GRANULARITY_BYTES = 1024; + // The number of bytes we have to waste in excess of our acceptable realtime rate of waste (defined below) + private static final long EXCESS_WASTE_BYTES = 10 * 1024 * 1024L; + private static final int EXCESS_WASTE_OFFSET = (int) (EXCESS_WASTE_BYTES / ALLOCATION_GRANULARITY_BYTES); + // Note this is a shift, because dividing a long time and then picking the low 32 bits doesn't give correct rollover behavior + private static final int CLOCK_SHIFT = 17; + // CLOCK_GRANULARITY = 1^9ns >> CLOCK_SHIFT == 132us == (1/7.63)ms + + /** + * (clock + allocation) granularity are combined to give us an acceptable (waste) allocation rate that is defined by + * the passage of real time of ALLOCATION_GRANULARITY_BYTES/CLOCK_GRANULARITY, or in this case 7.63Kb/ms, or 7.45Mb/s + * + * in wasteTracker we maintain within EXCESS_WASTE_OFFSET before the current time; whenever we waste bytes + * we increment the current value if it is within this window, and set it to the min of the window plus our waste + * otherwise. + */ + private volatile int wasteTracker = TRACKER_NEVER_WASTED; + + private static final AtomicIntegerFieldUpdater wasteTrackerUpdater = AtomicIntegerFieldUpdater.newUpdater(AtomicBTreePartition.class, "wasteTracker"); + + private static final DeletionInfo LIVE = DeletionInfo.live(); + // This is a small optimization: DeletionInfo is mutable, but we know that we will always copy it in that class, + // so we can safely alias one DeletionInfo.live() reference and avoid some allocations. + private static final Holder EMPTY = new Holder(BTree.empty(), LIVE, null, RowStats.NO_STATS); + + private final CFMetaData metadata; + private final DecoratedKey partitionKey; + private final MemtableAllocator allocator; + + private volatile Holder ref; + + private static final AtomicReferenceFieldUpdater refUpdater = AtomicReferenceFieldUpdater.newUpdater(AtomicBTreePartition.class, Holder.class, "ref"); + + public AtomicBTreePartition(CFMetaData metadata, DecoratedKey partitionKey, MemtableAllocator allocator) + { + this.metadata = metadata; + this.partitionKey = partitionKey; + this.allocator = allocator; + this.ref = EMPTY; + } + + public boolean isEmpty() + { + return ref.deletionInfo.isLive() && BTree.isEmpty(ref.tree) && ref.staticRow == null; + } + + public CFMetaData metadata() + { + return metadata; + } + + public DecoratedKey partitionKey() + { + return partitionKey; + } + + public DeletionTime partitionLevelDeletion() + { + return ref.deletionInfo.getPartitionDeletion(); + } + + public PartitionColumns columns() + { + // We don't really know which columns will be part of the update, so assume it's all of them + return metadata.partitionColumns(); + } + + public boolean hasRows() + { + return !BTree.isEmpty(ref.tree); + } + + public RowStats stats() + { + return ref.stats; + } + + public Row getRow(Clustering clustering) + { + Row row = searchIterator(ColumnFilter.selection(columns()), false).next(clustering); + // Note that for statics, this will never return null, this will return an empty row. However, + // it's more consistent for this method to return null if we don't really have a static row. + return row == null || (clustering == Clustering.STATIC_CLUSTERING && row.isEmpty()) ? null : row; + } + + public SearchIterator searchIterator(final ColumnFilter columns, final boolean reversed) + { + // TODO: we could optimize comparison for "NativeRow" à la #6755 + final Holder current = ref; + return new SearchIterator() + { + private final SearchIterator rawIter = new BTreeSearchIterator<>(current.tree, metadata.comparator, !reversed); + private final MemtableRowData.ReusableRow row = allocator.newReusableRow(); + private final ReusableFilteringRow filter = new ReusableFilteringRow(columns.fetchedColumns().regulars, columns); + private final long partitionDeletion = current.deletionInfo.getPartitionDeletion().markedForDeleteAt(); + + public boolean hasNext() + { + return rawIter.hasNext(); + } + + public Row next(Clustering key) + { + if (key == Clustering.STATIC_CLUSTERING) + return makeStatic(columns, current, allocator); + + MemtableRowData data = rawIter.next(key); + // We also need to find if there is a range tombstone covering this key + RangeTombstone rt = current.deletionInfo.rangeCovering(key); + + if (data == null) + { + // If we have a range tombstone but not data, "fake" the RT by return a row deletion + // corresponding to the tombstone. + if (rt != null && rt.deletionTime().markedForDeleteAt() > partitionDeletion) + return filter.setRowDeletion(rt.deletionTime()).setTo(emptyDeletedRow(key, rt.deletionTime())); + return null; + } + + row.setTo(data); + + filter.setRowDeletion(null); + if (rt == null || rt.deletionTime().markedForDeleteAt() < partitionDeletion) + { + filter.setDeletionTimestamp(partitionDeletion); + } + else + { + filter.setDeletionTimestamp(rt.deletionTime().markedForDeleteAt()); + // If we have a range tombstone covering that row and it's bigger than the row deletion itself, then + // we replace the row deletion by the tombstone deletion as a way to return the tombstone. + if (rt.deletionTime().supersedes(row.deletion())) + filter.setRowDeletion(rt.deletionTime()); + } + + return filter.setTo(row); + } + }; + } + + private static Row emptyDeletedRow(Clustering clustering, DeletionTime deletion) + { + return new AbstractRow() + { + public Columns columns() + { + return Columns.NONE; + } + + public LivenessInfo primaryKeyLivenessInfo() + { + return LivenessInfo.NONE; + } + + public DeletionTime deletion() + { + return deletion; + } + + public boolean isEmpty() + { + return true; + } + + public boolean hasComplexDeletion() + { + return false; + } + + public Clustering clustering() + { + return clustering; + } + + public Cell getCell(ColumnDefinition c) + { + return null; + } + + public Cell getCell(ColumnDefinition c, CellPath path) + { + return null; + } + + public Iterator getCells(ColumnDefinition c) + { + return null; + } + + public DeletionTime getDeletion(ColumnDefinition c) + { + return DeletionTime.LIVE; + } + + public Iterator iterator() + { + return Iterators.emptyIterator(); + } + + public SearchIterator searchIterator() + { + return new SearchIterator() + { + public boolean hasNext() + { + return false; + } + + public ColumnData next(ColumnDefinition column) + { + return null; + } + }; + } + + public Row takeAlias() + { + return this; + } + }; + } + + public UnfilteredRowIterator unfilteredIterator() + { + return unfilteredIterator(ColumnFilter.selection(columns()), Slices.ALL, false); + } + + public UnfilteredRowIterator unfilteredIterator(ColumnFilter selection, Slices slices, boolean reversed) + { + if (slices.size() == 0) + { + Holder current = ref; + DeletionTime partitionDeletion = current.deletionInfo.getPartitionDeletion(); + if (selection.fetchedColumns().statics.isEmpty() && partitionDeletion.isLive()) + return UnfilteredRowIterators.emptyIterator(metadata, partitionKey, reversed); + + return new AbstractUnfilteredRowIterator(metadata, + partitionKey, + partitionDeletion, + selection.fetchedColumns(), + makeStatic(selection, current, allocator), + reversed, + current.stats) + { + protected Unfiltered computeNext() + { + return endOfData(); + } + }; + } + + return slices.size() == 1 + ? new SingleSliceIterator(metadata, partitionKey, ref, selection, slices.get(0), reversed, allocator) + : new SlicesIterator(metadata, partitionKey, ref, selection, slices, reversed, allocator); + } + + private static Row makeStatic(ColumnFilter selection, Holder holder, MemtableAllocator allocator) + { + Columns statics = selection.fetchedColumns().statics; + if (statics.isEmpty() || holder.staticRow == null) + return Rows.EMPTY_STATIC_ROW; + + return new ReusableFilteringRow(statics, selection) + .setDeletionTimestamp(holder.deletionInfo.getPartitionDeletion().markedForDeleteAt()) + .setTo(allocator.newReusableRow().setTo(holder.staticRow)); + } + + private static class ReusableFilteringRow extends FilteringRow + { + private final Columns columns; + private final ColumnFilter selection; + private ColumnFilter.Tester tester; + private long deletionTimestamp; + + // Used by searchIterator in case the row is covered by a tombstone. + private DeletionTime rowDeletion; + + public ReusableFilteringRow(Columns columns, ColumnFilter selection) + { + this.columns = columns; + this.selection = selection; + } + + public ReusableFilteringRow setDeletionTimestamp(long timestamp) + { + this.deletionTimestamp = timestamp; + return this; + } + + public ReusableFilteringRow setRowDeletion(DeletionTime rowDeletion) + { + this.rowDeletion = rowDeletion; + return this; + } + + @Override + public DeletionTime deletion() + { + return rowDeletion == null ? super.deletion() : rowDeletion; + } + + @Override + protected boolean include(LivenessInfo info) + { + return info.timestamp() > deletionTimestamp; + } + + @Override + protected boolean include(ColumnDefinition def) + { + return columns.contains(def); + } + + @Override + protected boolean include(DeletionTime dt) + { + return dt.markedForDeleteAt() > deletionTimestamp; + } + + @Override + protected boolean include(ColumnDefinition c, DeletionTime dt) + { + return dt.markedForDeleteAt() > deletionTimestamp; + } + + @Override + protected boolean include(Cell cell) + { + return selection.includes(cell); + } + } + + private static class SingleSliceIterator extends AbstractUnfilteredRowIterator + { + private final Iterator iterator; + private final ReusableFilteringRow row; + + private SingleSliceIterator(CFMetaData metadata, + DecoratedKey key, + Holder holder, + ColumnFilter selection, + Slice slice, + boolean isReversed, + MemtableAllocator allocator) + { + super(metadata, + key, + holder.deletionInfo.getPartitionDeletion(), + selection.fetchedColumns(), + makeStatic(selection, holder, allocator), + isReversed, + holder.stats); + + Iterator rowIter = rowIter(metadata, + holder, + slice, + !isReversed, + allocator); + + this.iterator = new RowAndTombstoneMergeIterator(metadata.comparator, isReversed) + .setTo(rowIter, holder.deletionInfo.rangeIterator(slice, isReversed)); + + this.row = new ReusableFilteringRow(selection.fetchedColumns().regulars, selection) + .setDeletionTimestamp(partitionLevelDeletion.markedForDeleteAt()); + } + + private Iterator rowIter(CFMetaData metadata, + Holder holder, + Slice slice, + boolean forwards, + final MemtableAllocator allocator) + { + Slice.Bound start = slice.start() == Slice.Bound.BOTTOM ? null : slice.start(); + Slice.Bound end = slice.end() == Slice.Bound.TOP ? null : slice.end(); + final Iterator dataIter = BTree.slice(holder.tree, metadata.comparator, start, true, end, true, forwards); + return new AbstractIterator() + { + private final MemtableRowData.ReusableRow row = allocator.newReusableRow(); + + protected Row computeNext() + { + return dataIter.hasNext() ? row.setTo(dataIter.next()) : endOfData(); + } + }; + } + + protected Unfiltered computeNext() + { + while (iterator.hasNext()) + { + Unfiltered next = iterator.next(); + if (next.kind() == Unfiltered.Kind.ROW) + { + row.setTo((Row)next); + if (!row.isEmpty()) + return row; + } + else + { + RangeTombstoneMarker marker = (RangeTombstoneMarker)next; + + long deletion = partitionLevelDeletion().markedForDeleteAt(); + if (marker.isOpen(isReverseOrder())) + deletion = Math.max(deletion, marker.openDeletionTime(isReverseOrder()).markedForDeleteAt()); + row.setDeletionTimestamp(deletion); + return marker; + } + } + return endOfData(); + } + } + + public static class SlicesIterator extends AbstractUnfilteredRowIterator + { + private final Holder holder; + private final MemtableAllocator allocator; + private final ColumnFilter selection; + private final Slices slices; + + private int idx; + private UnfilteredRowIterator currentSlice; + + private SlicesIterator(CFMetaData metadata, + DecoratedKey key, + Holder holder, + ColumnFilter selection, + Slices slices, + boolean isReversed, + MemtableAllocator allocator) + { + super(metadata, key, holder.deletionInfo.getPartitionDeletion(), selection.fetchedColumns(), makeStatic(selection, holder, allocator), isReversed, holder.stats); + this.holder = holder; + this.selection = selection; + this.allocator = allocator; + this.slices = slices; + } + + protected Unfiltered computeNext() + { + while (true) + { + if (currentSlice == null) + { + if (idx >= slices.size()) + return endOfData(); + + int sliceIdx = isReverseOrder ? slices.size() - idx - 1 : idx; + currentSlice = new SingleSliceIterator(metadata, + partitionKey, + holder, + selection, + slices.get(sliceIdx), + isReverseOrder, + allocator); + idx++; + } + + if (currentSlice.hasNext()) + return currentSlice.next(); + + currentSlice = null; + } + } + } + + /** + * Adds a given update to this in-memtable partition. + * + * @return an array containing first the difference in size seen after merging the updates, and second the minimum + * time detla between updates. + */ + public long[] addAllWithSizeDelta(final PartitionUpdate update, OpOrder.Group writeOp, Updater indexer) + { + RowUpdater updater = new RowUpdater(this, allocator, writeOp, indexer); + DeletionInfo inputDeletionInfoCopy = null; + + boolean monitorOwned = false; + try + { + if (usePessimisticLocking()) + { + Locks.monitorEnterUnsafe(this); + monitorOwned = true; + } + while (true) + { + Holder current = ref; + updater.ref = current; + updater.reset(); + + DeletionInfo deletionInfo; + if (update.deletionInfo().mayModify(current.deletionInfo)) + { + if (inputDeletionInfoCopy == null) + inputDeletionInfoCopy = update.deletionInfo().copy(HeapAllocator.instance); + + deletionInfo = current.deletionInfo.copy().add(inputDeletionInfoCopy); + updater.allocated(deletionInfo.unsharedHeapSize() - current.deletionInfo.unsharedHeapSize()); + } + else + { + deletionInfo = current.deletionInfo; + } + + Row newStatic = update.staticRow(); + MemtableRowData staticRow = newStatic == Rows.EMPTY_STATIC_ROW + ? current.staticRow + : (current.staticRow == null ? updater.apply(newStatic) : updater.apply(current.staticRow, newStatic)); + Object[] tree = BTree.update(current.tree, update.metadata().comparator, update, update.rowCount(), updater); + RowStats newStats = current.stats.mergeWith(update.stats()); + + if (tree != null && refUpdater.compareAndSet(this, current, new Holder(tree, deletionInfo, staticRow, newStats))) + { + indexer.updateRowLevelIndexes(); + updater.finish(); + return new long[]{ updater.dataSize, updater.colUpdateTimeDelta }; + } + else if (!monitorOwned) + { + boolean shouldLock = usePessimisticLocking(); + if (!shouldLock) + { + shouldLock = updateWastedAllocationTracker(updater.heapSize); + } + if (shouldLock) + { + Locks.monitorEnterUnsafe(this); + monitorOwned = true; + } + } + } + } + finally + { + if (monitorOwned) + Locks.monitorExitUnsafe(this); + } + + } + + public boolean usePessimisticLocking() + { + return wasteTracker == TRACKER_PESSIMISTIC_LOCKING; + } + + /** + * Update the wasted allocation tracker state based on newly wasted allocation information + * + * @param wastedBytes the number of bytes wasted by this thread + * @return true if the caller should now proceed with pessimistic locking because the waste limit has been reached + */ + private boolean updateWastedAllocationTracker(long wastedBytes) + { + // Early check for huge allocation that exceeds the limit + if (wastedBytes < EXCESS_WASTE_BYTES) + { + // We round up to ensure work < granularity are still accounted for + int wastedAllocation = ((int) (wastedBytes + ALLOCATION_GRANULARITY_BYTES - 1)) / ALLOCATION_GRANULARITY_BYTES; + + int oldTrackerValue; + while (TRACKER_PESSIMISTIC_LOCKING != (oldTrackerValue = wasteTracker)) + { + // Note this time value has an arbitrary offset, but is a constant rate 32 bit counter (that may wrap) + int time = (int) (System.nanoTime() >>> CLOCK_SHIFT); + int delta = oldTrackerValue - time; + if (oldTrackerValue == TRACKER_NEVER_WASTED || delta >= 0 || delta < -EXCESS_WASTE_OFFSET) + delta = -EXCESS_WASTE_OFFSET; + delta += wastedAllocation; + if (delta >= 0) + break; + if (wasteTrackerUpdater.compareAndSet(this, oldTrackerValue, avoidReservedValues(time + delta))) + return false; + } + } + // We have definitely reached our waste limit so set the state if it isn't already + wasteTrackerUpdater.set(this, TRACKER_PESSIMISTIC_LOCKING); + // And tell the caller to proceed with pessimistic locking + return true; + } + + private static int avoidReservedValues(int wasteTracker) + { + if (wasteTracker == TRACKER_NEVER_WASTED || wasteTracker == TRACKER_PESSIMISTIC_LOCKING) + return wasteTracker + 1; + return wasteTracker; + } + + private static final class Holder + { + final DeletionInfo deletionInfo; + // the btree of rows + final Object[] tree; + final MemtableRowData staticRow; + final RowStats stats; + + Holder(Object[] tree, DeletionInfo deletionInfo, MemtableRowData staticRow, RowStats stats) + { + this.tree = tree; + this.deletionInfo = deletionInfo; + this.staticRow = staticRow; + this.stats = stats; + } + + Holder with(DeletionInfo info) + { + return new Holder(this.tree, info, this.staticRow, this.stats); + } + } + + // the function we provide to the btree utilities to perform any column replacements + private static final class RowUpdater implements UpdateFunction + { + final AtomicBTreePartition updating; + final MemtableAllocator allocator; + final OpOrder.Group writeOp; + final Updater indexer; + final int nowInSec; + Holder ref; + long dataSize; + long heapSize; + long colUpdateTimeDelta = Long.MAX_VALUE; + final MemtableRowData.ReusableRow row; + final MemtableAllocator.DataReclaimer reclaimer; + final MemtableAllocator.RowAllocator rowAllocator; + List inserted; // TODO: replace with walk of aborted BTree + + private RowUpdater(AtomicBTreePartition updating, MemtableAllocator allocator, OpOrder.Group writeOp, Updater indexer) + { + this.updating = updating; + this.allocator = allocator; + this.writeOp = writeOp; + this.indexer = indexer; + this.nowInSec = FBUtilities.nowInSeconds(); + this.row = allocator.newReusableRow(); + this.reclaimer = allocator.reclaimer(); + this.rowAllocator = allocator.newRowAllocator(updating.metadata(), writeOp); + } + + public MemtableRowData apply(Row insert) + { + rowAllocator.allocateNewRow(insert.clustering().size(), insert.columns(), insert.isStatic()); + insert.copyTo(rowAllocator); + MemtableRowData data = rowAllocator.allocatedRowData(); + + insertIntoIndexes(insert); + + this.dataSize += data.dataSize(); + this.heapSize += data.unsharedHeapSizeExcludingData(); + if (inserted == null) + inserted = new ArrayList<>(); + inserted.add(data); + return data; + } + + public MemtableRowData apply(MemtableRowData existing, Row update) + { + Columns mergedColumns = existing.columns().mergeTo(update.columns()); + rowAllocator.allocateNewRow(update.clustering().size(), mergedColumns, update.isStatic()); + + colUpdateTimeDelta = Math.min(colUpdateTimeDelta, Rows.merge(row.setTo(existing), update, mergedColumns, rowAllocator, nowInSec, indexer)); + + MemtableRowData reconciled = rowAllocator.allocatedRowData(); + + dataSize += reconciled.dataSize() - existing.dataSize(); + heapSize += reconciled.unsharedHeapSizeExcludingData() - existing.unsharedHeapSizeExcludingData(); + if (inserted == null) + inserted = new ArrayList<>(); + inserted.add(reconciled); + discard(existing); + + return reconciled; + } + + private void insertIntoIndexes(Row toInsert) + { + if (indexer == SecondaryIndexManager.nullUpdater) + return; + + maybeIndexPrimaryKeyColumns(toInsert); + Clustering clustering = toInsert.clustering(); + for (Cell cell : toInsert) + indexer.insert(clustering, cell); + } + + private void maybeIndexPrimaryKeyColumns(Row row) + { + // We want to update a primary key index with the most up to date info contains in that inserted row (if only for + // backward compatibility). Note that if there is an index but not a partition key or clustering column one, we've + // wasting this work. We might be able to avoid that if row indexing was pushed in the index updater. + long timestamp = row.primaryKeyLivenessInfo().timestamp(); + int ttl = row.primaryKeyLivenessInfo().ttl(); + + for (Cell cell : row) + { + long cellTimestamp = cell.livenessInfo().timestamp(); + if (cell.isLive(nowInSec)) + { + if (cellTimestamp > timestamp) + { + timestamp = cellTimestamp; + ttl = cell.livenessInfo().ttl(); + } + } + } + + indexer.maybeIndex(row.clustering(), timestamp, ttl, row.deletion()); + } + + protected void reset() + { + this.dataSize = 0; + this.heapSize = 0; + if (inserted != null) + { + for (MemtableRowData row : inserted) + abort(row); + inserted.clear(); + } + reclaimer.cancel(); + } + + protected void abort(MemtableRowData abort) + { + reclaimer.reclaimImmediately(abort); + } + + protected void discard(MemtableRowData discard) + { + reclaimer.reclaim(discard); + } + + public boolean abortEarly() + { + return updating.ref != ref; + } + + public void allocated(long heapSize) + { + this.heapSize += heapSize; + } + + protected void finish() + { + allocator.onHeap().allocate(heapSize, writeOp); + reclaimer.commit(); + } + } +} diff --git a/src/java/org/apache/cassandra/db/partitions/CachedPartition.java b/src/java/org/apache/cassandra/db/partitions/CachedPartition.java new file mode 100644 index 0000000000..dc482f3b61 --- /dev/null +++ b/src/java/org/apache/cassandra/db/partitions/CachedPartition.java @@ -0,0 +1,96 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.cassandra.db.partitions; + +import org.apache.cassandra.cache.IRowCacheEntry; +import org.apache.cassandra.db.rows.Row; +import org.apache.cassandra.io.ISerializer; + +/** + * A partition stored in the partition cache. + * + * Note that in practice, the only implementation of this is {@link ArrayBackedPartition}, + * we keep this interface mainly 1) to make it clear what we need from partition in the cache + * (that we don't otherwise) and 2) because {@code ArrayBackedPartition} is used for other + * purpose (than caching) and hence using {@code CachedPartition} when we talk about caching is + * clearer. + */ +public interface CachedPartition extends Partition, IRowCacheEntry +{ + public static final ISerializer cacheSerializer = new ArrayBackedCachedPartition.Serializer(); + + /** + * The number of {@code Row} objects in this cached partition. + * + * Please note that this is not the number of live rows since + * some of the row may only contains deleted (or expired) information. + * + * @return the number of row in the partition. + */ + public int rowCount(); + + /** + * The number of rows that were live at the time the partition was cached. + * + * See {@link ColumnFamilyStore#isFilterFullyCoveredBy} to see why we need this. + * + * @return the number of rows in this partition that were live at the time the + * partition was cached (this can be different from the number of live rows now + * due to expiring cells). + */ + public int cachedLiveRows(); + + /** + * The number of rows in this cached partition that have at least one non-expiring + * non-deleted cell. + * + * Note that this is generally not a very meaningful number, but this is used by + * {@link DataLimits#hasEnoughLiveData} as an optimization. + * + * @return the number of row that have at least one non-expiring non-deleted cell. + */ + public int rowsWithNonExpiringCells(); + + /** + * The last row in this cached partition (in order words, the row with the + * biggest clustering that the partition contains). + * + * @return the last row of the partition, or {@code null} if the partition is empty. + */ + public Row lastRow(); + + /** + * The number of {@code cell} objects that are not tombstone in this cached partition. + * + * Please note that this is not the number of live cells since + * some of the cells might be expired. + * + * @return the number of non tombstone cells in the partition. + */ + public int nonTombstoneCellCount(); + + /** + * The number of cells in this cached partition that are neither tombstone nor expiring. + * + * Note that this is generally not a very meaningful number, but this is used by + * {@link DataLimits#hasEnoughLiveData} as an optimization. + * + * @return the number of cells that are neither tombstones nor expiring. + */ + public int nonExpiringLiveCells(); +} diff --git a/src/java/org/apache/cassandra/db/partitions/CountingPartitionIterator.java b/src/java/org/apache/cassandra/db/partitions/CountingPartitionIterator.java new file mode 100644 index 0000000000..16445e7839 --- /dev/null +++ b/src/java/org/apache/cassandra/db/partitions/CountingPartitionIterator.java @@ -0,0 +1,58 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.cassandra.db.partitions; + +import org.apache.cassandra.db.rows.*; +import org.apache.cassandra.db.filter.DataLimits; + +public class CountingPartitionIterator extends WrappingPartitionIterator +{ + protected final DataLimits.Counter counter; + + public CountingPartitionIterator(PartitionIterator result, DataLimits.Counter counter) + { + super(result); + this.counter = counter; + } + + public CountingPartitionIterator(PartitionIterator result, DataLimits limits, int nowInSec) + { + this(result, limits.newCounter(nowInSec, true)); + } + + public DataLimits.Counter counter() + { + return counter; + } + + @Override + public boolean hasNext() + { + if (counter.isDone()) + return false; + + return super.hasNext(); + } + + @Override + @SuppressWarnings("resource") // Close through the closing of the returned 'CountingRowIterator' (and CountingRowIterator shouldn't throw) + public RowIterator next() + { + return new CountingRowIterator(super.next(), counter); + } +} diff --git a/src/java/org/apache/cassandra/db/composites/SimpleSparseInternedCellName.java b/src/java/org/apache/cassandra/db/partitions/CountingRowIterator.java similarity index 54% rename from src/java/org/apache/cassandra/db/composites/SimpleSparseInternedCellName.java rename to src/java/org/apache/cassandra/db/partitions/CountingRowIterator.java index c61372046e..4ad321e939 100644 --- a/src/java/org/apache/cassandra/db/composites/SimpleSparseInternedCellName.java +++ b/src/java/org/apache/cassandra/db/partitions/CountingRowIterator.java @@ -15,37 +15,44 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -package org.apache.cassandra.db.composites; +package org.apache.cassandra.db.partitions; -import org.apache.cassandra.config.CFMetaData; -import org.apache.cassandra.cql3.ColumnIdentifier; -import org.apache.cassandra.utils.memory.AbstractAllocator; +import org.apache.cassandra.db.rows.*; +import org.apache.cassandra.db.filter.DataLimits; -public class SimpleSparseInternedCellName extends SimpleSparseCellName +public class CountingRowIterator extends WrappingRowIterator { + protected final DataLimits.Counter counter; - // Not meant to be used directly, you should use the CellNameType method instead - SimpleSparseInternedCellName(ColumnIdentifier columnName) + public CountingRowIterator(RowIterator iter, DataLimits.Counter counter) { - super(columnName); + super(iter); + this.counter = counter; + + counter.newPartition(iter.partitionKey(), iter.staticRow()); } @Override - public long unsharedHeapSizeExcludingData() + public boolean hasNext() { - return 0; + if (counter.isDoneForPartition()) + return false; + + return super.hasNext(); } @Override - public long unsharedHeapSize() + public Row next() { - return 0; + Row row = super.next(); + counter.newRow(row); + return row; } @Override - public CellName copy(CFMetaData cfm, AbstractAllocator allocator) + public void close() { - // We're interning those instance in SparceCellNameType so don't need to copy. - return this; + super.close(); + counter.endOfPartition(); } } diff --git a/src/java/org/apache/cassandra/db/partitions/CountingUnfilteredPartitionIterator.java b/src/java/org/apache/cassandra/db/partitions/CountingUnfilteredPartitionIterator.java new file mode 100644 index 0000000000..52eedd4db4 --- /dev/null +++ b/src/java/org/apache/cassandra/db/partitions/CountingUnfilteredPartitionIterator.java @@ -0,0 +1,52 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.cassandra.db.partitions; + +import org.apache.cassandra.db.rows.*; +import org.apache.cassandra.db.filter.DataLimits; + +public class CountingUnfilteredPartitionIterator extends WrappingUnfilteredPartitionIterator +{ + protected final DataLimits.Counter counter; + + public CountingUnfilteredPartitionIterator(UnfilteredPartitionIterator result, DataLimits.Counter counter) + { + super(result); + this.counter = counter; + } + + public DataLimits.Counter counter() + { + return counter; + } + + @Override + public boolean hasNext() + { + if (counter.isDone()) + return false; + + return super.hasNext(); + } + + @Override + public UnfilteredRowIterator computeNext(UnfilteredRowIterator iter) + { + return new CountingUnfilteredRowIterator(iter, counter); + } +} diff --git a/src/java/org/apache/cassandra/db/partitions/CountingUnfilteredRowIterator.java b/src/java/org/apache/cassandra/db/partitions/CountingUnfilteredRowIterator.java new file mode 100644 index 0000000000..acaef5d96d --- /dev/null +++ b/src/java/org/apache/cassandra/db/partitions/CountingUnfilteredRowIterator.java @@ -0,0 +1,64 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.cassandra.db.partitions; + +import org.apache.cassandra.db.rows.*; +import org.apache.cassandra.db.filter.DataLimits; + +public class CountingUnfilteredRowIterator extends WrappingUnfilteredRowIterator +{ + private final DataLimits.Counter counter; + + public CountingUnfilteredRowIterator(UnfilteredRowIterator iter, DataLimits.Counter counter) + { + super(iter); + this.counter = counter; + + counter.newPartition(iter.partitionKey(), iter.staticRow()); + } + + public DataLimits.Counter counter() + { + return counter; + } + + @Override + public boolean hasNext() + { + if (counter.isDoneForPartition()) + return false; + + return super.hasNext(); + } + + @Override + public Unfiltered next() + { + Unfiltered unfiltered = super.next(); + if (unfiltered.kind() == Unfiltered.Kind.ROW) + counter.newRow((Row) unfiltered); + return unfiltered; + } + + @Override + public void close() + { + super.close(); + counter.endOfPartition(); + } +} diff --git a/src/java/org/apache/cassandra/db/partitions/FilteredPartition.java b/src/java/org/apache/cassandra/db/partitions/FilteredPartition.java new file mode 100644 index 0000000000..813654dde5 --- /dev/null +++ b/src/java/org/apache/cassandra/db/partitions/FilteredPartition.java @@ -0,0 +1,142 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.cassandra.db.partitions; + +import java.util.Iterator; + +import org.apache.cassandra.config.CFMetaData; +import org.apache.cassandra.db.*; +import org.apache.cassandra.db.rows.*; + +public class FilteredPartition extends AbstractPartitionData implements Iterable +{ + private FilteredPartition(CFMetaData metadata, + DecoratedKey partitionKey, + PartitionColumns columns, + int initialRowCapacity, + boolean sortable) + { + super(metadata, partitionKey, DeletionTime.LIVE, columns, initialRowCapacity, sortable); + } + + /** + * Create a FilteredPartition holding all the rows of the provided iterator. + * + * Warning: Note that this method does not close the provided iterator and it is + * up to the caller to do so. + */ + public static FilteredPartition create(RowIterator iterator) + { + FilteredPartition partition = new FilteredPartition(iterator.metadata(), + iterator.partitionKey(), + iterator.columns(), + 4, + iterator.isReverseOrder()); + + partition.staticRow = iterator.staticRow().takeAlias(); + + Writer writer = partition.new Writer(true); + + while (iterator.hasNext()) + iterator.next().copyTo(writer); + + // A Partition (or more precisely AbstractPartitionData) always assumes that its data is in clustering + // order. So if we've just added them in reverse clustering order, reverse them. + if (iterator.isReverseOrder()) + partition.reverse(); + + return partition; + } + + public RowIterator rowIterator() + { + final Iterator iter = iterator(); + return new RowIterator() + { + public CFMetaData metadata() + { + return metadata; + } + + public boolean isReverseOrder() + { + return false; + } + + public PartitionColumns columns() + { + return columns; + } + + public DecoratedKey partitionKey() + { + return key; + } + + public Row staticRow() + { + return staticRow == null ? Rows.EMPTY_STATIC_ROW : staticRow; + } + + public boolean hasNext() + { + return iter.hasNext(); + } + + public Row next() + { + return iter.next(); + } + + public void remove() + { + throw new UnsupportedOperationException(); + } + + public void close() + { + } + }; + } + + @Override + public String toString() + { + try (RowIterator iterator = rowIterator()) + { + StringBuilder sb = new StringBuilder(); + CFMetaData metadata = iterator.metadata(); + PartitionColumns columns = iterator.columns(); + + sb.append(String.format("[%s.%s] key=%s columns=%s reversed=%b", + metadata.ksName, + metadata.cfName, + metadata.getKeyValidator().getString(iterator.partitionKey().getKey()), + columns, + iterator.isReverseOrder())); + + if (iterator.staticRow() != Rows.EMPTY_STATIC_ROW) + sb.append("\n ").append(iterator.staticRow().toString(metadata)); + + while (iterator.hasNext()) + sb.append("\n ").append(iterator.next().toString(metadata)); + + return sb.toString(); + } + } +} diff --git a/src/java/org/apache/cassandra/db/partitions/FilteringPartitionIterator.java b/src/java/org/apache/cassandra/db/partitions/FilteringPartitionIterator.java new file mode 100644 index 0000000000..c40109b396 --- /dev/null +++ b/src/java/org/apache/cassandra/db/partitions/FilteringPartitionIterator.java @@ -0,0 +1,146 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.cassandra.db.partitions; + +import org.apache.cassandra.db.*; +import org.apache.cassandra.db.rows.*; + +/** + * Abstract class to make it easier to write iterators that filter some + * parts of another iterator (used for purging tombstones and removing dropped columns). + */ +public abstract class FilteringPartitionIterator extends WrappingUnfilteredPartitionIterator +{ + private UnfilteredRowIterator next; + + protected FilteringPartitionIterator(UnfilteredPartitionIterator iter) + { + super(iter); + } + + // The filter to use for filtering row contents. Is null by default to mean no particular filtering + // but can be overriden by subclasses. Please see FilteringAtomIterator for details on how this is used. + protected FilteringRow makeRowFilter() + { + return null; + } + + // Whether or not we should bother filtering the provided rows iterator. This + // exists mainly for preformance + protected boolean shouldFilter(UnfilteredRowIterator iterator) + { + return true; + } + + protected boolean includeRangeTombstoneMarker(RangeTombstoneMarker marker) + { + return true; + } + + protected boolean includePartitionDeletion(DeletionTime dt) + { + return true; + } + + // Allows to modify the range tombstone returned. This is called *after* includeRangeTombstoneMarker has been called. + protected RangeTombstoneMarker filterRangeTombstoneMarker(RangeTombstoneMarker marker, boolean reversed) + { + return marker; + } + + // Called when a particular partition is skipped due to being empty post filtering + protected void onEmpty(DecoratedKey key) + { + } + + public boolean hasNext() + { + while (next == null && super.hasNext()) + { + UnfilteredRowIterator iterator = super.next(); + if (shouldFilter(iterator)) + { + next = new FilteringIterator(iterator); + if (!isForThrift() && next.isEmpty()) + { + onEmpty(iterator.partitionKey()); + iterator.close(); + next = null; + } + } + else + { + next = iterator; + } + } + return next != null; + } + + public UnfilteredRowIterator next() + { + UnfilteredRowIterator toReturn = next; + next = null; + return toReturn; + } + + @Override + public void close() + { + try + { + super.close(); + } + finally + { + if (next != null) + next.close(); + } + } + + private class FilteringIterator extends FilteringRowIterator + { + private FilteringIterator(UnfilteredRowIterator iterator) + { + super(iterator); + } + + @Override + protected FilteringRow makeRowFilter() + { + return FilteringPartitionIterator.this.makeRowFilter(); + } + + @Override + protected boolean includeRangeTombstoneMarker(RangeTombstoneMarker marker) + { + return FilteringPartitionIterator.this.includeRangeTombstoneMarker(marker); + } + + @Override + protected RangeTombstoneMarker filterRangeTombstoneMarker(RangeTombstoneMarker marker, boolean reversed) + { + return FilteringPartitionIterator.this.filterRangeTombstoneMarker(marker, reversed); + } + + @Override + protected boolean includePartitionDeletion(DeletionTime dt) + { + return FilteringPartitionIterator.this.includePartitionDeletion(dt); + } + } +} diff --git a/src/java/org/apache/cassandra/db/partitions/Partition.java b/src/java/org/apache/cassandra/db/partitions/Partition.java new file mode 100644 index 0000000000..71d041153a --- /dev/null +++ b/src/java/org/apache/cassandra/db/partitions/Partition.java @@ -0,0 +1,70 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.cassandra.db.partitions; + +import org.apache.cassandra.config.CFMetaData; +import org.apache.cassandra.db.Slices; +import org.apache.cassandra.db.*; +import org.apache.cassandra.db.rows.*; +import org.apache.cassandra.db.filter.ColumnFilter; +import org.apache.cassandra.utils.SearchIterator; + +/** + * In-memory representation of a Partition. + * + * Note that most of the storage engine works through iterators (UnfilteredPartitionIterator) to + * avoid "materializing" a full partition/query response in memory as much as possible, + * and so Partition objects should be use as sparingly as possible. There is a couple + * of cases where we do need to represent partition in-memory (memtables and row cache). + */ +public interface Partition +{ + public CFMetaData metadata(); + public DecoratedKey partitionKey(); + public DeletionTime partitionLevelDeletion(); + + public PartitionColumns columns(); + + public RowStats stats(); + + /** + * Whether the partition object has no informations at all, including any deletion informations. + */ + public boolean isEmpty(); + + /** + * Returns the row corresponding to the provided clustering, or null if there is not such row. + */ + public Row getRow(Clustering clustering); + + /** + * Returns an iterator that allows to search specific rows efficiently. + */ + public SearchIterator searchIterator(ColumnFilter columns, boolean reversed); + + /** + * Returns an UnfilteredRowIterator over all the rows/RT contained by this partition. + */ + public UnfilteredRowIterator unfilteredIterator(); + + /** + * Returns an UnfilteredRowIterator over the rows/RT contained by this partition + * selected by the provided slices. + */ + public UnfilteredRowIterator unfilteredIterator(ColumnFilter columns, Slices slices, boolean reversed); +} diff --git a/src/java/org/apache/cassandra/service/pager/Pageable.java b/src/java/org/apache/cassandra/db/partitions/PartitionIterator.java similarity index 51% rename from src/java/org/apache/cassandra/service/pager/Pageable.java rename to src/java/org/apache/cassandra/db/partitions/PartitionIterator.java index d4986f7acc..36358fc69d 100644 --- a/src/java/org/apache/cassandra/service/pager/Pageable.java +++ b/src/java/org/apache/cassandra/db/partitions/PartitionIterator.java @@ -15,27 +15,25 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -package org.apache.cassandra.service.pager; +package org.apache.cassandra.db.partitions; -import java.util.List; +import java.util.Iterator; -import org.apache.cassandra.db.ReadCommand; +import org.apache.cassandra.db.rows.*; /** - * Marker interface for commands that can be paged. + * An iterator over a number of (filtered) partition. + * + * PartitionIterator is to RowIterator what UnfilteredPartitionIterator is to UnfilteredRowIterator + * though unlike UnfilteredPartitionIterator, it is not guaranteed that the RowIterator + * returned are in partitioner order. + * + * The object returned by a call to next() is only guaranteed to be + * valid until the next call to hasNext() or next(). If a consumer wants to keep a + * reference on the returned objects for longer than the iteration, it must + * make a copy of it explicitely. */ -public interface Pageable +public interface PartitionIterator extends Iterator, AutoCloseable { - public static class ReadCommands implements Pageable - { - public final List commands; - - public final int limitForQuery; - - public ReadCommands(List commands, int limitForQuery) - { - this.commands = commands; - this.limitForQuery = limitForQuery; - } - } + public void close(); } diff --git a/src/java/org/apache/cassandra/db/partitions/PartitionIterators.java b/src/java/org/apache/cassandra/db/partitions/PartitionIterators.java new file mode 100644 index 0000000000..219aa5a436 --- /dev/null +++ b/src/java/org/apache/cassandra/db/partitions/PartitionIterators.java @@ -0,0 +1,198 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.cassandra.db.partitions; + +import java.util.*; +import java.security.MessageDigest; + +import com.google.common.collect.AbstractIterator; + +import org.apache.cassandra.db.SinglePartitionReadCommand; +import org.apache.cassandra.db.rows.*; +import org.apache.cassandra.io.util.FileUtils; + +public abstract class PartitionIterators +{ + private PartitionIterators() {} + + public static final PartitionIterator EMPTY = new PartitionIterator() + { + public boolean hasNext() + { + return false; + } + + public RowIterator next() + { + throw new NoSuchElementException(); + } + + public void remove() + { + } + + public void close() + { + } + }; + + @SuppressWarnings("resource") // The created resources are returned right away + public static RowIterator getOnlyElement(final PartitionIterator iter, SinglePartitionReadCommand command) + { + // If the query has no results, we'll get an empty iterator, but we still + // want a RowIterator out of this method, so we return an empty one. + RowIterator toReturn = iter.hasNext() + ? iter.next() + : RowIterators.emptyIterator(command.metadata(), + command.partitionKey(), + command.clusteringIndexFilter().isReversed()); + + // Note that in general, we should wrap the result so that it's close method actually + // close the whole PartitionIterator. + return new WrappingRowIterator(toReturn) + { + public void close() + { + try + { + super.close(); + } + finally + { + // asserting this only now because it bothers UnfilteredPartitionIterators.Serializer (which might be used + // under the provided DataIter) if hasNext() is called before the previously returned iterator hasn't been fully consumed. + assert !iter.hasNext(); + + iter.close(); + } + } + }; + } + + @SuppressWarnings("resource") // The created resources are returned right away + public static PartitionIterator concat(final List iterators) + { + if (iterators.size() == 1) + return iterators.get(0); + + return new PartitionIterator() + { + private int idx = 0; + + public boolean hasNext() + { + while (idx < iterators.size()) + { + if (iterators.get(idx).hasNext()) + return true; + + ++idx; + } + return false; + } + + public RowIterator next() + { + if (!hasNext()) + throw new NoSuchElementException(); + return iterators.get(idx).next(); + } + + public void remove() + { + throw new UnsupportedOperationException(); + } + + public void close() + { + FileUtils.closeQuietly(iterators); + } + }; + } + + public static void digest(PartitionIterator iterator, MessageDigest digest) + { + while (iterator.hasNext()) + { + try (RowIterator partition = iterator.next()) + { + RowIterators.digest(partition, digest); + } + } + } + + public static PartitionIterator singletonIterator(RowIterator iterator) + { + return new SingletonPartitionIterator(iterator); + } + + public static void consume(PartitionIterator iterator) + { + while (iterator.hasNext()) + { + try (RowIterator partition = iterator.next()) + { + while (partition.hasNext()) + partition.next(); + } + } + } + + /** + * Wraps the provided iterator so it logs the returned rows for debugging purposes. + *

+ * Note that this is only meant for debugging as this can log a very large amount of + * logging at INFO. + */ + @SuppressWarnings("resource") // The created resources are returned right away + public static PartitionIterator loggingIterator(PartitionIterator iterator, final String id) + { + return new WrappingPartitionIterator(iterator) + { + public RowIterator next() + { + return RowIterators.loggingIterator(super.next(), id); + } + }; + } + + private static class SingletonPartitionIterator extends AbstractIterator implements PartitionIterator + { + private final RowIterator iterator; + private boolean returned; + + private SingletonPartitionIterator(RowIterator iterator) + { + this.iterator = iterator; + } + + protected RowIterator computeNext() + { + if (returned) + return endOfData(); + + returned = true; + return iterator; + } + + public void close() + { + iterator.close(); + } + } +} diff --git a/src/java/org/apache/cassandra/db/partitions/PartitionUpdate.java b/src/java/org/apache/cassandra/db/partitions/PartitionUpdate.java new file mode 100644 index 0000000000..ca1e424b51 --- /dev/null +++ b/src/java/org/apache/cassandra/db/partitions/PartitionUpdate.java @@ -0,0 +1,764 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.cassandra.db.partitions; + +import java.io.DataInput; +import java.io.DataInputStream; +import java.io.IOException; +import java.nio.ByteBuffer; +import java.util.*; + +import com.google.common.collect.Iterables; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import org.apache.cassandra.config.CFMetaData; +import org.apache.cassandra.config.ColumnDefinition; +import org.apache.cassandra.db.*; +import org.apache.cassandra.db.filter.ColumnFilter; +import org.apache.cassandra.db.rows.*; +import org.apache.cassandra.db.index.SecondaryIndexManager; +import org.apache.cassandra.io.util.DataOutputBuffer; +import org.apache.cassandra.io.util.DataOutputPlus; +import org.apache.cassandra.net.MessagingService; +import org.apache.cassandra.utils.ByteBufferUtil; +import org.apache.cassandra.utils.FBUtilities; +import org.apache.cassandra.utils.Sorting; + +/** + * Stores updates made on a partition. + *

+ * A PartitionUpdate object requires that all writes are performed before we + * try to read the updates (attempts to write to the PartitionUpdate after a + * read method has been called will result in an exception being thrown). + * In other words, a Partition is mutable while we do a write and become + * immutable as soon as it is read. + *

+ * Row updates are added to this update through the {@link #writer} method which + * returns a {@link Row.Writer}. Multiple rows can be added to this writer as required and + * those row do not have to added in (clustering) order, and the same row can be added + * multiple times. Further, for a given row, the writer actually supports intermingling + * the writing of cells for different complex cells (note that this is usually not supported + * by {@code Row.Writer} implementations, but is supported here because + * {@code ModificationStatement} requires that (because we could have multiple {@link Operation} + * on the same column in a given statement)). + */ +public class PartitionUpdate extends AbstractPartitionData implements Sorting.Sortable +{ + protected static final Logger logger = LoggerFactory.getLogger(PartitionUpdate.class); + + // Records whether the partition update has been sorted (it is the rows contained in the partition + // that are sorted since we don't require rows to be added in order). Sorting happens when the + // update is read, and writting is rejected as soon as the update is sorted (it's actually possible + // to manually allow new update by using allowNewUpdates(), and we could make that more implicit, but + // as only triggers really requires it, we keep it simple for now). + private boolean isSorted; + + public static final PartitionUpdateSerializer serializer = new PartitionUpdateSerializer(); + + private final Writer writer; + + // Used by compare for the sake of implementing the Sorting.Sortable interface (which is in turn used + // to sort the rows of this update). + private final InternalReusableClustering p1 = new InternalReusableClustering(); + private final InternalReusableClustering p2 = new InternalReusableClustering(); + + private PartitionUpdate(CFMetaData metadata, + DecoratedKey key, + DeletionInfo delInfo, + RowDataBlock data, + PartitionColumns columns, + int initialRowCapacity) + { + super(metadata, key, delInfo, columns, data, initialRowCapacity); + this.writer = createWriter(); + } + + public PartitionUpdate(CFMetaData metadata, + DecoratedKey key, + DeletionInfo delInfo, + PartitionColumns columns, + int initialRowCapacity) + { + this(metadata, + key, + delInfo, + new RowDataBlock(columns.regulars, initialRowCapacity, true, metadata.isCounter()), + columns, + initialRowCapacity); + } + + public PartitionUpdate(CFMetaData metadata, + DecoratedKey key, + PartitionColumns columns, + int initialRowCapacity) + { + this(metadata, + key, + DeletionInfo.live(), + columns, + initialRowCapacity); + } + + protected Writer createWriter() + { + return new RegularWriter(); + } + + protected StaticWriter createStaticWriter() + { + return new StaticWriter(); + } + + /** + * Deserialize a partition update from a provided byte buffer. + * + * @param bytes the byte buffer that contains the serialized update. + * @param version the version with which the update is serialized. + * @param key the partition key for the update. This is only used if {@code version < 3.0} + * and can be {@code null} otherwise. + * + * @return the deserialized update or {@code null} if {@code bytes == null}. + */ + public static PartitionUpdate fromBytes(ByteBuffer bytes, int version, DecoratedKey key) + { + if (bytes == null) + return null; + + try + { + return serializer.deserialize(new DataInputStream(ByteBufferUtil.inputStream(bytes)), + version, + SerializationHelper.Flag.LOCAL, + version < MessagingService.VERSION_30 ? key : null); + } + catch (IOException e) + { + throw new RuntimeException(e); + } + } + + /** + * Serialize a partition update as a byte buffer. + * + * @param update the partition update to serialize. + * @param version the version to serialize the update into. + * + * @return a newly allocated byte buffer containing the serialized update. + */ + public static ByteBuffer toBytes(PartitionUpdate update, int version) + { + try (DataOutputBuffer out = new DataOutputBuffer()) + { + serializer.serialize(update, out, MessagingService.current_version); + return ByteBuffer.wrap(out.getData(), 0, out.getLength()); + } + catch (IOException e) + { + throw new RuntimeException(e); + } + } + + /** + * Creates a empty immutable partition update. + * + * @param metadata the metadata for the created update. + * @param key the partition key for the created update. + * + * @return the newly created empty (and immutable) update. + */ + public static PartitionUpdate emptyUpdate(CFMetaData metadata, DecoratedKey key) + { + return new PartitionUpdate(metadata, key, PartitionColumns.NONE, 0) + { + public Row.Writer staticWriter() + { + throw new UnsupportedOperationException(); + } + + public Row.Writer writer() + { + throw new UnsupportedOperationException(); + } + + public void addPartitionDeletion(DeletionTime deletionTime) + { + throw new UnsupportedOperationException(); + } + + public void addRangeTombstone(RangeTombstone range) + { + throw new UnsupportedOperationException(); + } + }; + } + + /** + * Creates a partition update that entirely deletes a given partition. + * + * @param metadata the metadata for the created update. + * @param key the partition key for the partition that the created update should delete. + * @param timestamp the timestamp for the deletion. + * @param nowInSec the current time in seconds to use as local deletion time for the partition deletion. + * + * @return the newly created partition deletion update. + */ + public static PartitionUpdate fullPartitionDelete(CFMetaData metadata, DecoratedKey key, long timestamp, int nowInSec) + { + return new PartitionUpdate(metadata, + key, + new DeletionInfo(timestamp, nowInSec), + new RowDataBlock(Columns.NONE, 0, true, metadata.isCounter()), + PartitionColumns.NONE, + 0); + } + + /** + * Merges the provided updates, yielding a new update that incorporates all those updates. + * + * @param updates the collection of updates to merge. This shouldn't be empty. + * + * @return a partition update that include (merge) all the updates from {@code updates}. + */ + public static PartitionUpdate merge(Collection updates) + { + assert !updates.isEmpty(); + if (updates.size() == 1) + return Iterables.getOnlyElement(updates); + + int totalSize = 0; + PartitionColumns.Builder builder = PartitionColumns.builder(); + DecoratedKey key = null; + CFMetaData metadata = null; + for (PartitionUpdate update : updates) + { + totalSize += update.rows; + builder.addAll(update.columns()); + + if (key == null) + key = update.partitionKey(); + else + assert key.equals(update.partitionKey()); + + if (metadata == null) + metadata = update.metadata(); + else + assert metadata.cfId.equals(update.metadata().cfId); + } + + // Used when merging row to decide of liveness + int nowInSec = FBUtilities.nowInSeconds(); + PartitionUpdate newUpdate = new PartitionUpdate(metadata, key, builder.build(), totalSize); + for (PartitionUpdate update : updates) + { + newUpdate.deletionInfo.add(update.deletionInfo); + if (!update.staticRow().isEmpty()) + { + if (newUpdate.staticRow().isEmpty()) + newUpdate.staticRow = update.staticRow().takeAlias(); + else + Rows.merge(newUpdate.staticRow(), update.staticRow(), newUpdate.columns().statics, newUpdate.staticWriter(), nowInSec, SecondaryIndexManager.nullUpdater); + } + for (Row row : update) + row.copyTo(newUpdate.writer); + } + return newUpdate; + } + + /** + * The number of "operations" contained in the update. + *

+ * This is used by {@code Memtable} to approximate how much work this update does. In practice, this + * count how many rows are updated and how many ranges are deleted by the partition update. + * + * @return the number of "operations" performed by the update. + */ + public int operationCount() + { + return rowCount() + + deletionInfo.rangeCount() + + (deletionInfo.getPartitionDeletion().isLive() ? 0 : 1); + } + + /** + * The size of the data contained in this update. + * + * @return the size of the data contained in this update. + */ + public int dataSize() + { + int clusteringSize = metadata().comparator.size(); + int size = 0; + for (Row row : this) + { + size += row.clustering().dataSize(); + for (Cell cell : row) + size += cell.dataSize(); + } + return size; + } + + /** + * If a partition update has been read (and is thus unmodifiable), a call to this method + * makes the update modifiable again. + *

+ * Please note that calling this method won't result in optimal behavior in the sense that + * even if very little is added to the update after this call, the whole update will be sorted + * again on read. This should thus be used sparingly (and if it turns that we end up using + * this often, we should consider optimizing the behavior). + */ + public synchronized void allowNewUpdates() + { + // This is synchronized to make extra sure things work properly even if this is + // called concurrently with sort() (which should be avoided in the first place, but + // better safe than sorry). + isSorted = false; + } + + /** + * Returns an iterator that iterators over the rows of this update in clustering order. + *

+ * Note that this might trigger a sorting of the update, and as such the update will not + * be modifiable anymore after this call. + * + * @return an iterator over the rows of this update. + */ + @Override + public Iterator iterator() + { + maybeSort(); + return super.iterator(); + } + + @Override + protected SliceableUnfilteredRowIterator sliceableUnfilteredIterator(ColumnFilter columns, boolean reversed) + { + maybeSort(); + return super.sliceableUnfilteredIterator(columns, reversed); + } + + /** + * Validates the data contained in this update. + * + * @throws MarshalException if some of the data contained in this update is corrupted. + */ + public void validate() + { + for (Row row : this) + { + metadata().comparator.validate(row.clustering()); + for (Cell cell : row) + cell.validate(); + } + } + + /** + * The maximum timestamp used in this update. + * + * @return the maximum timestamp used in this update. + */ + public long maxTimestamp() + { + return maxTimestamp; + } + + /** + * For an update on a counter table, returns a list containing a {@code CounterMark} for + * every counter contained in the update. + * + * @return a list with counter marks for every counter in this update. + */ + public List collectCounterMarks() + { + assert metadata().isCounter(); + + InternalReusableClustering clustering = new InternalReusableClustering(); + List l = new ArrayList<>(); + int i = 0; + for (Row row : this) + { + for (Cell cell : row) + if (cell.isCounterCell()) + l.add(new CounterMark(clustering, i, cell.column(), cell.path())); + i++; + } + return l; + } + + /** + * Returns a row writer for the static row of this partition update. + * + * @return a row writer for the static row of this partition update. A partition + * update contains only one static row so only one row should be written through + * this writer (but if multiple rows are added, the latest written one wins). + */ + public Row.Writer staticWriter() + { + return createStaticWriter(); + } + + /** + * Returns a row writer to add (non-static) rows to this partition update. + * + * @return a row writer to add (non-static) rows to this partition update. + * Multiple rows can be successively added this way and the rows added do not have + * to be in clustering order. Further, the same row can be added multiple time. + * + */ + public Row.Writer writer() + { + if (isSorted) + throw new IllegalStateException("An update should not written again once it has been read"); + + return writer; + } + + /** + * Returns a range tombstone marker writer to add range tombstones to this + * partition update. + *

+ * Note that if more convenient, range tombstones can also be added using + * {@link addRangeTombstone}. + * + * @param isReverseOrder whether the range tombstone marker will be provided to the returned writer + * in clustering order or in reverse clustering order. + * @return a range tombstone marker writer to add range tombstones to this update. + */ + public RangeTombstoneMarker.Writer markerWriter(boolean isReverseOrder) + { + return new RangeTombstoneCollector(isReverseOrder); + } + + /** + * The number of rows contained in this update. + * + * @return the number of rows contained in this update. + */ + public int size() + { + return rows; + } + + private void maybeSort() + { + if (isSorted) + return; + + sort(); + } + + private synchronized void sort() + { + if (isSorted) + return; + + if (rows <= 1) + { + isSorted = true; + return; + } + + // Sort the rows - will still potentially contain duplicate (non-reconciled) rows + Sorting.sort(this); + + // Now find duplicates and merge them together + int previous = 0; // The last element that was set + int nowInSec = FBUtilities.nowInSeconds(); + for (int current = 1; current < rows; current++) + { + // There is really only 2 possible comparison: < 0 or == 0 since we've sorted already + int cmp = compare(previous, current); + if (cmp == 0) + { + // current and previous are the same row. Merge current into previous + // (and so previous + 1 will be "free"). + data.merge(current, previous, nowInSec); + } + else + { + // data[current] != [previous], so move current just after previous if needs be + ++previous; + if (previous != current) + data.move(current, previous); + } + } + + // previous is on the last value to keep + rows = previous + 1; + + isSorted = true; + } + + /** + * This method is note meant to be used externally: it is only public so this + * update conform to the {@link Sorting.Sortable} interface. + */ + public int compare(int i, int j) + { + return metadata.comparator.compare(p1.setTo(i), p2.setTo(j)); + } + + protected class StaticWriter extends StaticRow.Builder + { + protected StaticWriter() + { + super(columns.statics, false, metadata().isCounter()); + } + + @Override + public void endOfRow() + { + super.endOfRow(); + if (staticRow == null) + { + staticRow = build(); + } + else + { + StaticRow.Builder builder = StaticRow.builder(columns.statics, true, metadata().isCounter()); + Rows.merge(staticRow, build(), columns.statics, builder, FBUtilities.nowInSeconds()); + staticRow = builder.build(); + } + } + } + + protected class RegularWriter extends Writer + { + // For complex column, the writer assumptions is that for a given row, cells of different + // complex columns are not intermingled (they also should be in cellPath order). We however + // don't yet guarantee that this will be the case for updates (both UpdateStatement and + // RowUpdateBuilder can potentially break that assumption; we could change those classes but + // that's non trivial, at least for UpdateStatement). + // To deal with that problem, we record which complex columns have been updated (for the current + // row) and if we detect a violation of our assumption, we switch the row we're writing + // into (which is ok because everything will be sorted and merged in maybeSort()). + private final Set updatedComplex = new HashSet(); + private ColumnDefinition lastUpdatedComplex; + private CellPath lastUpdatedComplexPath; + + public RegularWriter() + { + super(false); + } + + @Override + public void writeCell(ColumnDefinition column, boolean isCounter, ByteBuffer value, LivenessInfo info, CellPath path) + { + if (column.isComplex()) + { + if (updatedComplex.contains(column) + && (!column.equals(lastUpdatedComplex) || (column.cellPathComparator().compare(path, lastUpdatedComplexPath)) <= 0)) + { + // We've updated that complex already, but we've either updated another complex or it's not in order: as this + // break the writer assumption, switch rows. + endOfRow(); + + // Copy the clustering values from the previous row + int clusteringSize = metadata.clusteringColumns().size(); + int base = (row - 1) * clusteringSize; + for (int i = 0; i < clusteringSize; i++) + writer.writeClusteringValue(clusterings[base + i]); + + updatedComplex.clear(); + } + + lastUpdatedComplex = column; + lastUpdatedComplexPath = path; + updatedComplex.add(column); + } + super.writeCell(column, isCounter, value, info, path); + } + + @Override + public void endOfRow() + { + super.endOfRow(); + clear(); + } + + @Override + public Writer reset() + { + super.reset(); + clear(); + return this; + } + + private void clear() + { + updatedComplex.clear(); + lastUpdatedComplex = null; + lastUpdatedComplexPath = null; + } + } + + public static class PartitionUpdateSerializer + { + public void serialize(PartitionUpdate update, DataOutputPlus out, int version) throws IOException + { + if (version < MessagingService.VERSION_30) + { + // TODO + throw new UnsupportedOperationException(); + + // if (cf == null) + // { + // out.writeBoolean(false); + // return; + // } + + // out.writeBoolean(true); + // serializeCfId(cf.id(), out, version); + // cf.getComparator().deletionInfoSerializer().serialize(cf.deletionInfo(), out, version); + // ColumnSerializer columnSerializer = cf.getComparator().columnSerializer(); + // int count = cf.getColumnCount(); + // out.writeInt(count); + // int written = 0; + // for (Cell cell : cf) + // { + // columnSerializer.serialize(cell, out); + // written++; + // } + // assert count == written: "Table had " + count + " columns, but " + written + " written"; + } + + try (UnfilteredRowIterator iter = update.sliceableUnfilteredIterator()) + { + assert !iter.isReverseOrder(); + UnfilteredRowIteratorSerializer.serializer.serialize(iter, out, version, update.rows); + } + } + + public PartitionUpdate deserialize(DataInput in, int version, SerializationHelper.Flag flag, DecoratedKey key) throws IOException + { + if (version < MessagingService.VERSION_30) + { + assert key != null; + + // This is only used in mutation, and mutation have never allowed "null" column families + boolean present = in.readBoolean(); + assert present; + + CFMetaData metadata = CFMetaData.serializer.deserialize(in, version); + LegacyLayout.LegacyDeletionInfo info = LegacyLayout.LegacyDeletionInfo.serializer.deserialize(metadata, in, version); + int size = in.readInt(); + Iterator cells = LegacyLayout.deserializeCells(metadata, in, flag, size); + SerializationHelper helper = new SerializationHelper(version, flag); + try (UnfilteredRowIterator iterator = LegacyLayout.onWireCellstoUnfilteredRowIterator(metadata, key, info, cells, false, helper)) + { + return UnfilteredRowIterators.toUpdate(iterator); + } + } + + assert key == null; // key is only there for the old format + + UnfilteredRowIteratorSerializer.Header h = UnfilteredRowIteratorSerializer.serializer.deserializeHeader(in, version, flag); + if (h.isEmpty) + return emptyUpdate(h.metadata, h.key); + + assert !h.isReversed; + assert h.rowEstimate >= 0; + PartitionUpdate upd = new PartitionUpdate(h.metadata, + h.key, + new DeletionInfo(h.partitionDeletion), + new RowDataBlock(h.sHeader.columns().regulars, h.rowEstimate, false, h.metadata.isCounter()), + h.sHeader.columns(), + h.rowEstimate); + + upd.staticRow = h.staticRow; + + RangeTombstoneMarker.Writer markerWriter = upd.markerWriter(false); + UnfilteredRowIteratorSerializer.serializer.deserialize(in, new SerializationHelper(version, flag), h.sHeader, upd.writer(), markerWriter); + + // Mark sorted after we're read it all since that's what we use in the writer() method to detect bad uses + upd.isSorted = true; + + return upd; + } + + public long serializedSize(PartitionUpdate update, int version, TypeSizes sizes) + { + if (version < MessagingService.VERSION_30) + { + // TODO + throw new UnsupportedOperationException("Version is " + version); + //if (cf == null) + //{ + // return typeSizes.sizeof(false); + //} + //else + //{ + // return typeSizes.sizeof(true) /* nullness bool */ + // + cfIdSerializedSize(cf.id(), typeSizes, version) /* id */ + // + contentSerializedSize(cf, typeSizes, version); + //} + } + + try (UnfilteredRowIterator iter = update.sliceableUnfilteredIterator()) + { + return UnfilteredRowIteratorSerializer.serializer.serializedSize(iter, version, update.rows, sizes); + } + } + } + + /** + * A counter mark is basically a pointer to a counter update inside this partition update. That pointer allows + * us to update the counter value based on the pre-existing value read during the read-before-write that counters + * do. See {@link CounterMutation} to understand how this is used. + */ + public class CounterMark + { + private final InternalReusableClustering clustering; + private final int row; + private final ColumnDefinition column; + private final CellPath path; + + private CounterMark(InternalReusableClustering clustering, int row, ColumnDefinition column, CellPath path) + { + this.clustering = clustering; + this.row = row; + this.column = column; + this.path = path; + } + + public Clustering clustering() + { + return clustering.setTo(row); + } + + public ColumnDefinition column() + { + return column; + } + + public CellPath path() + { + return path; + } + + public ByteBuffer value() + { + return data.getValue(row, column, path); + } + + public void setValue(ByteBuffer value) + { + data.setValue(row, column, path, value); + } + } +} diff --git a/src/java/org/apache/cassandra/db/partitions/SingletonUnfilteredPartitionIterator.java b/src/java/org/apache/cassandra/db/partitions/SingletonUnfilteredPartitionIterator.java new file mode 100644 index 0000000000..e2fec0597b --- /dev/null +++ b/src/java/org/apache/cassandra/db/partitions/SingletonUnfilteredPartitionIterator.java @@ -0,0 +1,64 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.cassandra.db.partitions; + +import java.util.NoSuchElementException; + +import org.apache.cassandra.db.rows.UnfilteredRowIterator; + +public class SingletonUnfilteredPartitionIterator implements UnfilteredPartitionIterator +{ + private final UnfilteredRowIterator iter; + private final boolean isForThrift; + private boolean returned; + + public SingletonUnfilteredPartitionIterator(UnfilteredRowIterator iter, boolean isForThrift) + { + this.iter = iter; + this.isForThrift = isForThrift; + } + + public boolean isForThrift() + { + return isForThrift; + } + + public boolean hasNext() + { + return !returned; + } + + public UnfilteredRowIterator next() + { + if (returned) + throw new NoSuchElementException(); + + returned = true; + return iter; + } + + public void remove() + { + throw new UnsupportedOperationException(); + } + + public void close() + { + iter.close(); + } +} diff --git a/src/java/org/apache/cassandra/db/partitions/TombstonePurgingPartitionIterator.java b/src/java/org/apache/cassandra/db/partitions/TombstonePurgingPartitionIterator.java new file mode 100644 index 0000000000..10022eb44c --- /dev/null +++ b/src/java/org/apache/cassandra/db/partitions/TombstonePurgingPartitionIterator.java @@ -0,0 +1,103 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.cassandra.db.partitions; + +import org.apache.cassandra.config.ColumnDefinition; +import org.apache.cassandra.db.*; +import org.apache.cassandra.db.rows.*; + +public abstract class TombstonePurgingPartitionIterator extends FilteringPartitionIterator +{ + private final int gcBefore; + + public TombstonePurgingPartitionIterator(UnfilteredPartitionIterator iterator, int gcBefore) + { + super(iterator); + this.gcBefore = gcBefore; + } + + protected abstract long getMaxPurgeableTimestamp(); + + protected FilteringRow makeRowFilter() + { + return new FilteringRow() + { + @Override + protected boolean include(LivenessInfo info) + { + return !info.hasLocalDeletionTime() || !info.isPurgeable(getMaxPurgeableTimestamp(), gcBefore); + } + + @Override + protected boolean include(DeletionTime dt) + { + return includeDelTime(dt); + } + + @Override + protected boolean include(ColumnDefinition c, DeletionTime dt) + { + return includeDelTime(dt); + } + }; + } + + private boolean includeDelTime(DeletionTime dt) + { + return dt.isLive() || !dt.isPurgeable(getMaxPurgeableTimestamp(), gcBefore); + } + + @Override + protected boolean includePartitionDeletion(DeletionTime dt) + { + return includeDelTime(dt); + } + + @Override + protected boolean includeRangeTombstoneMarker(RangeTombstoneMarker marker) + { + if (marker.isBoundary()) + { + // We can only skip the whole marker if both deletion time are purgeable. + // If only one of them is, filterTombstoneMarker will deal with it. + RangeTombstoneBoundaryMarker boundary = (RangeTombstoneBoundaryMarker)marker; + return includeDelTime(boundary.endDeletionTime()) || includeDelTime(boundary.startDeletionTime()); + } + else + { + return includeDelTime(((RangeTombstoneBoundMarker)marker).deletionTime()); + } + } + + @Override + protected RangeTombstoneMarker filterRangeTombstoneMarker(RangeTombstoneMarker marker, boolean reversed) + { + if (!marker.isBoundary()) + return marker; + + // Note that we know this is called after includeRangeTombstoneMarker. So if one of the deletion time is + // purgeable, we know the other one isn't. + RangeTombstoneBoundaryMarker boundary = (RangeTombstoneBoundaryMarker)marker; + if (!(includeDelTime(boundary.closeDeletionTime(reversed)))) + return boundary.createCorrespondingCloseBound(reversed); + else if (!(includeDelTime(boundary.openDeletionTime(reversed)))) + return boundary.createCorrespondingOpenBound(reversed); + return boundary; + } + +}; diff --git a/src/java/org/apache/cassandra/db/partitions/UnfilteredPartitionIterator.java b/src/java/org/apache/cassandra/db/partitions/UnfilteredPartitionIterator.java new file mode 100644 index 0000000000..2447da8401 --- /dev/null +++ b/src/java/org/apache/cassandra/db/partitions/UnfilteredPartitionIterator.java @@ -0,0 +1,46 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.cassandra.db.partitions; + +import java.util.Iterator; + +import org.apache.cassandra.db.rows.UnfilteredRowIterator; + +/** + * An iterator over a number of unfiltered partitions (i.e. partitions containing deletion informations). + * + * The object returned by a call to next() is only guaranteed to be + * valid until the next call to hasNext() or next(). If a consumer wants to keep a + * reference on the returned objects for longer than the iteration, it must + * make a copy of it explicitely. + */ +public interface UnfilteredPartitionIterator extends Iterator, AutoCloseable +{ + /** + * Whether that partition iterator is for a thrift queries. + *

+ * If this is true, the partition iterator may return some empty UnfilteredRowIterator and those + * should be preserved as thrift include partitions that "exists" (have some cells even + * if this are actually deleted) but have nothing matching the query. + * + * @return whether the iterator is for a thrift query. + */ + public boolean isForThrift(); + + public void close(); +} diff --git a/src/java/org/apache/cassandra/db/partitions/UnfilteredPartitionIterators.java b/src/java/org/apache/cassandra/db/partitions/UnfilteredPartitionIterators.java new file mode 100644 index 0000000000..f66ec11931 --- /dev/null +++ b/src/java/org/apache/cassandra/db/partitions/UnfilteredPartitionIterators.java @@ -0,0 +1,503 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.cassandra.db.partitions; + +import java.io.DataInput; +import java.io.IOError; +import java.io.IOException; +import java.security.MessageDigest; +import java.util.*; + +import org.apache.cassandra.config.CFMetaData; +import org.apache.cassandra.config.ColumnDefinition; +import org.apache.cassandra.cql3.ColumnIdentifier; +import org.apache.cassandra.db.*; +import org.apache.cassandra.db.rows.*; +import org.apache.cassandra.io.util.DataOutputPlus; +import org.apache.cassandra.net.MessagingService; +import org.apache.cassandra.utils.MergeIterator; + +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +/** + * Static methods to work with partition iterators. + */ +public abstract class UnfilteredPartitionIterators +{ + private static final Logger logger = LoggerFactory.getLogger(UnfilteredPartitionIterators.class); + + private static final Serializer serializer = new Serializer(); + + private static final Comparator partitionComparator = new Comparator() + { + public int compare(UnfilteredRowIterator p1, UnfilteredRowIterator p2) + { + return p1.partitionKey().compareTo(p2.partitionKey()); + } + }; + + public static final UnfilteredPartitionIterator EMPTY = new AbstractUnfilteredPartitionIterator() + { + public boolean isForThrift() + { + return false; + } + + public boolean hasNext() + { + return false; + } + + public UnfilteredRowIterator next() + { + throw new NoSuchElementException(); + } + }; + + private UnfilteredPartitionIterators() {} + + public interface MergeListener + { + public UnfilteredRowIterators.MergeListener getRowMergeListener(DecoratedKey partitionKey, List versions); + public void close(); + } + + @SuppressWarnings("resource") // The created resources are returned right away + public static UnfilteredRowIterator getOnlyElement(final UnfilteredPartitionIterator iter, SinglePartitionReadCommand command) + { + // If the query has no results, we'll get an empty iterator, but we still + // want a RowIterator out of this method, so we return an empty one. + UnfilteredRowIterator toReturn = iter.hasNext() + ? iter.next() + : UnfilteredRowIterators.emptyIterator(command.metadata(), + command.partitionKey(), + command.clusteringIndexFilter().isReversed()); + + // Note that in general, we should wrap the result so that it's close method actually + // close the whole UnfilteredPartitionIterator. + return new WrappingUnfilteredRowIterator(toReturn) + { + public void close() + { + try + { + super.close(); + } + finally + { + // asserting this only now because it bothers Serializer if hasNext() is called before + // the previously returned iterator hasn't been fully consumed. + assert !iter.hasNext(); + + iter.close(); + } + } + }; + } + + public static PartitionIterator mergeAndFilter(List iterators, int nowInSec, MergeListener listener) + { + // TODO: we could have a somewhat faster version if we were to merge the UnfilteredRowIterators directly as RowIterators + return filter(merge(iterators, nowInSec, listener), nowInSec); + } + + public static PartitionIterator filter(final UnfilteredPartitionIterator iterator, final int nowInSec) + { + return new PartitionIterator() + { + private RowIterator next; + + public boolean hasNext() + { + while (next == null && iterator.hasNext()) + { + @SuppressWarnings("resource") // closed either directly if empty, or, if assigned to next, by either + // the caller of next() or close() + UnfilteredRowIterator rowIterator = iterator.next(); + next = UnfilteredRowIterators.filter(rowIterator, nowInSec); + if (!iterator.isForThrift() && next.isEmpty()) + { + rowIterator.close(); + next = null; + } + } + return next != null; + } + + public RowIterator next() + { + if (next == null && !hasNext()) + throw new NoSuchElementException(); + + RowIterator toReturn = next; + next = null; + return toReturn; + } + + public void remove() + { + throw new UnsupportedOperationException(); + } + + public void close() + { + try + { + iterator.close(); + } + finally + { + if (next != null) + next.close(); + } + } + }; + } + + public static UnfilteredPartitionIterator merge(final List iterators, final int nowInSec, final MergeListener listener) + { + assert listener != null; + assert !iterators.isEmpty(); + + final boolean isForThrift = iterators.get(0).isForThrift(); + + final MergeIterator merged = MergeIterator.get(iterators, partitionComparator, new MergeIterator.Reducer() + { + private final List toMerge = new ArrayList<>(iterators.size()); + + private CFMetaData metadata; + private DecoratedKey partitionKey; + private boolean isReverseOrder; + + public void reduce(int idx, UnfilteredRowIterator current) + { + metadata = current.metadata(); + partitionKey = current.partitionKey(); + isReverseOrder = current.isReverseOrder(); + + // Note that because the MergeListener cares about it, we want to preserve the index of the iterator. + // Non-present iterator will thus be set to empty in getReduced. + toMerge.set(idx, current); + } + + protected UnfilteredRowIterator getReduced() + { + UnfilteredRowIterators.MergeListener rowListener = listener.getRowMergeListener(partitionKey, toMerge); + + // Replace nulls by empty iterators + for (int i = 0; i < toMerge.size(); i++) + if (toMerge.get(i) == null) + toMerge.set(i, UnfilteredRowIterators.emptyIterator(metadata, partitionKey, isReverseOrder)); + + return UnfilteredRowIterators.merge(toMerge, nowInSec, rowListener); + } + + protected void onKeyChange() + { + toMerge.clear(); + for (int i = 0; i < iterators.size(); i++) + toMerge.add(null); + } + }); + + return new AbstractUnfilteredPartitionIterator() + { + public boolean isForThrift() + { + return isForThrift; + } + + public boolean hasNext() + { + return merged.hasNext(); + } + + public UnfilteredRowIterator next() + { + return merged.next(); + } + + @Override + public void close() + { + merged.close(); + } + }; + } + + /** + * Convert all expired cells to equivalent tombstones. + *

+ * See {@link UnfilteredRowIterators#convertExpiredCellsToTombstones} for details. + * + * @param iterator the iterator in which to conver expired cells. + * @param nowInSec the current time to use to decide if a cell is expired. + * @return an iterator that returns the same data than {@code iterator} but with all expired cells converted + * to equivalent tombstones. + */ + public static UnfilteredPartitionIterator convertExpiredCellsToTombstones(UnfilteredPartitionIterator iterator, final int nowInSec) + { + return new WrappingUnfilteredPartitionIterator(iterator) + { + @Override + protected UnfilteredRowIterator computeNext(UnfilteredRowIterator iter) + { + return UnfilteredRowIterators.convertExpiredCellsToTombstones(iter, nowInSec); + } + }; + } + + public static UnfilteredPartitionIterator mergeLazily(final List iterators, final int nowInSec) + { + assert !iterators.isEmpty(); + + if (iterators.size() == 1) + return iterators.get(0); + + final boolean isForThrift = iterators.get(0).isForThrift(); + + final MergeIterator merged = MergeIterator.get(iterators, partitionComparator, new MergeIterator.Reducer() + { + private final List toMerge = new ArrayList<>(iterators.size()); + + @Override + public boolean trivialReduceIsTrivial() + { + return false; + } + + public void reduce(int idx, UnfilteredRowIterator current) + { + toMerge.add(current); + } + + protected UnfilteredRowIterator getReduced() + { + return new LazilyInitializedUnfilteredRowIterator(toMerge.get(0).partitionKey()) + { + protected UnfilteredRowIterator initializeIterator() + { + return UnfilteredRowIterators.merge(toMerge, nowInSec); + } + }; + } + + protected void onKeyChange() + { + toMerge.clear(); + } + }); + + return new AbstractUnfilteredPartitionIterator() + { + public boolean isForThrift() + { + return isForThrift; + } + + public boolean hasNext() + { + return merged.hasNext(); + } + + public UnfilteredRowIterator next() + { + return merged.next(); + } + + @Override + public void close() + { + merged.close(); + } + }; + } + + public static UnfilteredPartitionIterator removeDroppedColumns(UnfilteredPartitionIterator iterator, final Map droppedColumns) + { + return new FilteringPartitionIterator(iterator) + { + @Override + protected FilteringRow makeRowFilter() + { + return new FilteringRow() + { + @Override + protected boolean include(Cell cell) + { + return include(cell.column(), cell.livenessInfo().timestamp()); + } + + @Override + protected boolean include(ColumnDefinition c, DeletionTime dt) + { + return include(c, dt.markedForDeleteAt()); + } + + private boolean include(ColumnDefinition column, long timestamp) + { + CFMetaData.DroppedColumn dropped = droppedColumns.get(column.name); + return dropped == null || timestamp > dropped.droppedTime; + } + }; + } + + @Override + protected boolean shouldFilter(UnfilteredRowIterator iterator) + { + // TODO: We could have row iterators return the smallest timestamp they might return + // (which we can get from sstable stats), and ignore any dropping if that smallest + // timestamp is bigger that the biggest droppedColumns timestamp. + + // If none of the dropped columns is part of the columns that the iterator actually returns, there is nothing to do; + for (ColumnDefinition c : iterator.columns()) + if (droppedColumns.containsKey(c.name)) + return true; + + return false; + } + }; + } + + public static void digest(UnfilteredPartitionIterator iterator, MessageDigest digest) + { + try (UnfilteredPartitionIterator iter = iterator) + { + while (iter.hasNext()) + { + try (UnfilteredRowIterator partition = iter.next()) + { + UnfilteredRowIterators.digest(partition, digest); + } + } + } + } + + public static Serializer serializerForIntraNode() + { + return serializer; + } + + /** + * Wraps the provided iterator so it logs the returned rows/RT for debugging purposes. + *

+ * Note that this is only meant for debugging as this can log a very large amount of + * logging at INFO. + */ + public static UnfilteredPartitionIterator loggingIterator(UnfilteredPartitionIterator iterator, final String id, final boolean fullDetails) + { + return new WrappingUnfilteredPartitionIterator(iterator) + { + public UnfilteredRowIterator next() + { + return UnfilteredRowIterators.loggingIterator(super.next(), id, fullDetails); + } + }; + } + + /** + * Serialize each UnfilteredSerializer one after the other, with an initial byte that indicates whether + * we're done or not. + */ + public static class Serializer + { + public void serialize(UnfilteredPartitionIterator iter, DataOutputPlus out, int version) throws IOException + { + if (version < MessagingService.VERSION_30) + throw new UnsupportedOperationException(); + + out.writeBoolean(iter.isForThrift()); + while (iter.hasNext()) + { + out.writeBoolean(true); + try (UnfilteredRowIterator partition = iter.next()) + { + UnfilteredRowIteratorSerializer.serializer.serialize(partition, out, version); + } + } + out.writeBoolean(false); + } + + public UnfilteredPartitionIterator deserialize(final DataInput in, final int version, final SerializationHelper.Flag flag) throws IOException + { + if (version < MessagingService.VERSION_30) + throw new UnsupportedOperationException(); + + final boolean isForThrift = in.readBoolean(); + + return new AbstractUnfilteredPartitionIterator() + { + private UnfilteredRowIterator next; + private boolean hasNext; + private boolean nextReturned = true; + + public boolean isForThrift() + { + return isForThrift; + } + + public boolean hasNext() + { + if (!nextReturned) + return hasNext; + + // We can't answer this until the previously returned iterator has been fully consumed, + // so complain if that's not the case. + if (next != null && next.hasNext()) + throw new IllegalStateException("Cannot call hasNext() until the previous iterator has been fully consumed"); + + try + { + hasNext = in.readBoolean(); + nextReturned = false; + return hasNext; + } + catch (IOException e) + { + throw new IOError(e); + } + } + + public UnfilteredRowIterator next() + { + if (nextReturned && !hasNext()) + throw new NoSuchElementException(); + + try + { + nextReturned = true; + next = UnfilteredRowIteratorSerializer.serializer.deserialize(in, version, flag); + return next; + } + catch (IOException e) + { + throw new IOError(e); + } + } + + @Override + public void close() + { + if (next != null) + next.close(); + } + }; + } + } +} diff --git a/src/java/org/apache/cassandra/cql3/CQL3Row.java b/src/java/org/apache/cassandra/db/partitions/WrappingPartitionIterator.java similarity index 59% rename from src/java/org/apache/cassandra/cql3/CQL3Row.java rename to src/java/org/apache/cassandra/db/partitions/WrappingPartitionIterator.java index e3e76d18e6..4d4be70727 100644 --- a/src/java/org/apache/cassandra/cql3/CQL3Row.java +++ b/src/java/org/apache/cassandra/db/partitions/WrappingPartitionIterator.java @@ -15,27 +15,36 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -package org.apache.cassandra.cql3; +package org.apache.cassandra.db.partitions; -import java.nio.ByteBuffer; -import java.util.Iterator; -import java.util.List; +import org.apache.cassandra.db.rows.RowIterator; -import org.apache.cassandra.db.Cell; - -public interface CQL3Row +public abstract class WrappingPartitionIterator implements PartitionIterator { - public ByteBuffer getClusteringColumn(int i); - public Cell getColumn(ColumnIdentifier name); - public List getMultiCellColumn(ColumnIdentifier name); + protected final PartitionIterator wrapped; - public interface Builder + protected WrappingPartitionIterator(PartitionIterator wrapped) { - public RowIterator group(Iterator cells); + this.wrapped = wrapped; } - public interface RowIterator extends Iterator + public boolean hasNext() { - public CQL3Row getStaticRow(); + return wrapped.hasNext(); + } + + public RowIterator next() + { + return wrapped.next(); + } + + public void remove() + { + wrapped.remove(); + } + + public void close() + { + wrapped.close(); } } diff --git a/src/java/org/apache/cassandra/db/partitions/WrappingUnfilteredPartitionIterator.java b/src/java/org/apache/cassandra/db/partitions/WrappingUnfilteredPartitionIterator.java new file mode 100644 index 0000000000..4f350756f9 --- /dev/null +++ b/src/java/org/apache/cassandra/db/partitions/WrappingUnfilteredPartitionIterator.java @@ -0,0 +1,120 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.cassandra.db.partitions; + +import org.apache.cassandra.db.rows.UnfilteredRowIterator; +import org.apache.cassandra.db.rows.UnfilteredRowIterators; + +/** + * A utility class for writing partition iterators that filter/modify other + * partition iterators. + * + * This work a little bit like Guava's AbstractIterator in that you only need + * to implement the computeNext() method, though that method takes as argument + * the UnfilteredRowIterator to filter from the wrapped partition iterator. + */ +public abstract class WrappingUnfilteredPartitionIterator extends AbstractUnfilteredPartitionIterator +{ + protected final UnfilteredPartitionIterator wrapped; + + private UnfilteredRowIterator next; + + protected WrappingUnfilteredPartitionIterator(UnfilteredPartitionIterator wrapped) + { + this.wrapped = wrapped; + } + + public boolean isForThrift() + { + return wrapped.isForThrift(); + } + + public boolean hasNext() + { + prepareNext(); + return next != null; + } + + public UnfilteredRowIterator next() + { + prepareNext(); + assert next != null; + + UnfilteredRowIterator toReturn = next; + next = null; + return toReturn; + } + + private void prepareNext() + { + while (next == null && wrapped.hasNext()) + { + @SuppressWarnings("resource") // Closed on exception, right away if empty or ignored by computeNext, or if assigned to 'next', + // either by the caller to next(), or in close(). + UnfilteredRowIterator wrappedNext = wrapped.next(); + try + { + UnfilteredRowIterator maybeNext = computeNext(wrappedNext); + + // As the wrappd iterator shouldn't return an empty iterator, if computeNext + // gave us back it's input we save the isEmpty check. + if (maybeNext != null && (isForThrift() || maybeNext == wrappedNext || !maybeNext.isEmpty())) + { + next = maybeNext; + return; + } + else + { + wrappedNext.close(); + } + } + catch (RuntimeException | Error e) + { + wrappedNext.close(); + throw e; + } + } + } + + /** + * Given the next UnfilteredRowIterator from the wrapped partition iterator, return + * the (potentially modified) UnfilteredRowIterator to return. Please note that the + * result will be skipped if it's either {@code null} of if it's empty. + * + * The default implementation return it's input unchanged to make it easier + * to write wrapping partition iterators that only change the close method. + */ + protected UnfilteredRowIterator computeNext(UnfilteredRowIterator iter) + { + return iter; + } + + @Override + public void close() + { + try + { + wrapped.close(); + } + finally + { + if (next != null) + next.close(); + } + } +} diff --git a/src/java/org/apache/cassandra/db/rows/AbstractCell.java b/src/java/org/apache/cassandra/db/rows/AbstractCell.java new file mode 100644 index 0000000000..c003d6f987 --- /dev/null +++ b/src/java/org/apache/cassandra/db/rows/AbstractCell.java @@ -0,0 +1,135 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.cassandra.db.rows; + +import java.security.MessageDigest; +import java.util.Objects; + +import org.apache.cassandra.db.context.CounterContext; +import org.apache.cassandra.db.marshal.AbstractType; +import org.apache.cassandra.db.marshal.CollectionType; +import org.apache.cassandra.serializers.MarshalException; +import org.apache.cassandra.utils.FBUtilities; + +/** + * Base abstract class for {@code Cell} implementations. + * + * Unless you have a very good reason not to, every cell implementation + * should probably extend this class. + */ +public abstract class AbstractCell implements Cell +{ + public boolean isLive(int nowInSec) + { + return livenessInfo().isLive(nowInSec); + } + + public boolean isTombstone() + { + return livenessInfo().hasLocalDeletionTime() && !livenessInfo().hasTTL(); + } + + public boolean isExpiring() + { + return livenessInfo().hasTTL(); + } + + public void writeTo(Row.Writer writer) + { + writer.writeCell(column(), isCounterCell(), value(), livenessInfo(), path()); + } + + public void digest(MessageDigest digest) + { + digest.update(value().duplicate()); + livenessInfo().digest(digest); + FBUtilities.updateWithBoolean(digest, isCounterCell()); + if (path() != null) + path().digest(digest); + } + + public void validate() + { + column().validateCellValue(value()); + + livenessInfo().validate(); + + // If cell is a tombstone, it shouldn't have a value. + if (isTombstone() && value().hasRemaining()) + throw new MarshalException("A tombstone should not have a value"); + + if (path() != null) + column().validateCellPath(path()); + } + + public int dataSize() + { + int size = value().remaining() + livenessInfo().dataSize(); + if (path() != null) + size += path().dataSize(); + return size; + + } + + @Override + public boolean equals(Object other) + { + if(!(other instanceof Cell)) + return false; + + Cell that = (Cell)other; + return this.column().equals(that.column()) + && this.isCounterCell() == that.isCounterCell() + && Objects.equals(this.value(), that.value()) + && Objects.equals(this.livenessInfo(), that.livenessInfo()) + && Objects.equals(this.path(), that.path()); + } + + @Override + public int hashCode() + { + return Objects.hash(column(), isCounterCell(), value(), livenessInfo(), path()); + } + + @Override + public String toString() + { + if (isCounterCell()) + return String.format("[%s=%d ts=%d]", column().name, CounterContext.instance().total(value()), livenessInfo().timestamp()); + + AbstractType type = column().type; + if (type instanceof CollectionType && type.isMultiCell()) + { + CollectionType ct = (CollectionType)type; + return String.format("[%s[%s]=%s info=%s]", + column().name, + ct.nameComparator().getString(path().get(0)), + ct.valueComparator().getString(value()), + livenessInfo()); + } + return String.format("[%s=%s info=%s]", column().name, type.getString(value()), livenessInfo()); + } + + public Cell takeAlias() + { + // Cell is always used as an Aliasable object but as the code currently + // never need to alias a cell outside of its valid scope, we don't yet + // need that. + throw new UnsupportedOperationException(); + } +} diff --git a/src/java/org/apache/cassandra/db/rows/AbstractRangeTombstoneMarker.java b/src/java/org/apache/cassandra/db/rows/AbstractRangeTombstoneMarker.java new file mode 100644 index 0000000000..d8256fccb7 --- /dev/null +++ b/src/java/org/apache/cassandra/db/rows/AbstractRangeTombstoneMarker.java @@ -0,0 +1,71 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.cassandra.db.rows; + +import java.nio.ByteBuffer; + +import org.apache.cassandra.config.CFMetaData; +import org.apache.cassandra.db.*; + +public abstract class AbstractRangeTombstoneMarker implements RangeTombstoneMarker +{ + protected final RangeTombstone.Bound bound; + + protected AbstractRangeTombstoneMarker(RangeTombstone.Bound bound) + { + this.bound = bound; + } + + public RangeTombstone.Bound clustering() + { + return bound; + } + + public Unfiltered.Kind kind() + { + return Unfiltered.Kind.RANGE_TOMBSTONE_MARKER; + } + + public void validateData(CFMetaData metadata) + { + Slice.Bound bound = clustering(); + for (int i = 0; i < bound.size(); i++) + { + ByteBuffer value = bound.get(i); + if (value != null) + metadata.comparator.subtype(i).validate(value); + } + } + + public String toString(CFMetaData metadata, boolean fullDetails) + { + return toString(metadata); + } + + protected void copyBoundTo(RangeTombstoneMarker.Writer writer) + { + for (int i = 0; i < bound.size(); i++) + writer.writeClusteringValue(bound.get(i)); + writer.writeBoundKind(bound.kind()); + } + + public Unfiltered takeAlias() + { + return this; + } +} diff --git a/src/java/org/apache/cassandra/db/rows/AbstractReusableRow.java b/src/java/org/apache/cassandra/db/rows/AbstractReusableRow.java new file mode 100644 index 0000000000..03aeb8886d --- /dev/null +++ b/src/java/org/apache/cassandra/db/rows/AbstractReusableRow.java @@ -0,0 +1,207 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.cassandra.db.rows; + +import java.util.Iterator; + +import org.apache.cassandra.config.ColumnDefinition; +import org.apache.cassandra.db.*; +import org.apache.cassandra.utils.SearchIterator; + +public abstract class AbstractReusableRow extends AbstractRow +{ + private CellData.ReusableCell simpleCell; + private ComplexRowDataBlock.ReusableIterator complexCells; + private DeletionTimeArray.Cursor complexDeletionCursor; + private RowDataBlock.ReusableIterator iterator; + + public AbstractReusableRow() + { + } + + protected abstract int row(); + protected abstract RowDataBlock data(); + + private CellData.ReusableCell simpleCell() + { + if (simpleCell == null) + simpleCell = SimpleRowDataBlock.reusableCell(); + return simpleCell; + } + + private ComplexRowDataBlock.ReusableIterator complexCells() + { + if (complexCells == null) + complexCells = ComplexRowDataBlock.reusableComplexCells(); + return complexCells; + } + + private DeletionTimeArray.Cursor complexDeletionCursor() + { + if (complexDeletionCursor == null) + complexDeletionCursor = ComplexRowDataBlock.complexDeletionCursor(); + return complexDeletionCursor; + } + + private RowDataBlock.ReusableIterator reusableIterator() + { + if (iterator == null) + iterator = RowDataBlock.reusableIterator(); + return iterator; + } + + public Columns columns() + { + return data().columns(); + } + + public Cell getCell(ColumnDefinition c) + { + assert !c.isComplex(); + if (data().simpleData == null) + return null; + + int idx = columns().simpleIdx(c, 0); + if (idx < 0) + return null; + + return simpleCell().setTo(data().simpleData.data, c, (row() * columns().simpleColumnCount()) + idx); + } + + public Cell getCell(ColumnDefinition c, CellPath path) + { + assert c.isComplex(); + + ComplexRowDataBlock data = data().complexData; + if (data == null) + return null; + + int idx = data.cellIdx(row(), c, path); + if (idx < 0) + return null; + + return simpleCell().setTo(data.cellData(row()), c, idx); + } + + public Iterator getCells(ColumnDefinition c) + { + assert c.isComplex(); + return complexCells().setTo(data().complexData, row(), c); + } + + public boolean hasComplexDeletion() + { + return data().hasComplexDeletion(row()); + } + + public DeletionTime getDeletion(ColumnDefinition c) + { + assert c.isComplex(); + if (data().complexData == null) + return DeletionTime.LIVE; + + int idx = data().complexData.complexDeletionIdx(row(), c); + return idx < 0 + ? DeletionTime.LIVE + : complexDeletionCursor().setTo(data().complexData.complexDelTimes, idx); + } + + public Iterator iterator() + { + return reusableIterator().setTo(data(), row()); + } + + public SearchIterator searchIterator() + { + return new SearchIterator() + { + private int simpleIdx = 0; + + public boolean hasNext() + { + // TODO: we can do better, but we expect users to no rely on this anyway + return true; + } + + public ColumnData next(ColumnDefinition column) + { + if (column.isComplex()) + { + // TODO: this is sub-optimal + + Iterator cells = getCells(column); + return cells == null ? null : new ColumnData(column, null, cells, getDeletion(column)); + } + else + { + int idx = columns().simpleIdx(column, simpleIdx); + if (idx < 0) + return null; + + Cell cell = simpleCell().setTo(data().simpleData.data, column, (row() * columns().simpleColumnCount()) + idx); + simpleIdx = idx + 1; + return cell == null ? null : new ColumnData(column, cell, null, null); + } + } + }; + } + + public Row takeAlias() + { + final Clustering clustering = clustering().takeAlias(); + final LivenessInfo info = primaryKeyLivenessInfo().takeAlias(); + final DeletionTime deletion = deletion().takeAlias(); + + final RowDataBlock data = data(); + final int row = row(); + + return new AbstractReusableRow() + { + protected RowDataBlock data() + { + return data; + } + + protected int row() + { + return row; + } + + public Clustering clustering() + { + return clustering; + } + + public LivenessInfo primaryKeyLivenessInfo() + { + return info; + } + + public DeletionTime deletion() + { + return deletion; + } + + @Override + public Row takeAlias() + { + return this; + } + }; + } +} diff --git a/src/java/org/apache/cassandra/db/rows/AbstractRow.java b/src/java/org/apache/cassandra/db/rows/AbstractRow.java new file mode 100644 index 0000000000..a99bc7807e --- /dev/null +++ b/src/java/org/apache/cassandra/db/rows/AbstractRow.java @@ -0,0 +1,209 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.cassandra.db.rows; + +import java.nio.ByteBuffer; +import java.security.MessageDigest; +import java.util.Iterator; +import java.util.Objects; + +import org.apache.cassandra.config.CFMetaData; +import org.apache.cassandra.config.ColumnDefinition; +import org.apache.cassandra.db.*; +import org.apache.cassandra.db.marshal.CollectionType; +import org.apache.cassandra.serializers.MarshalException; +import org.apache.cassandra.utils.FBUtilities; + +/** + * Base abstract class for {@code Row} implementations. + * + * Unless you have a very good reason not to, every row implementation + * should probably extend this class. + */ +public abstract class AbstractRow implements Row +{ + public Unfiltered.Kind kind() + { + return Unfiltered.Kind.ROW; + } + + public boolean hasLiveData(int nowInSec) + { + if (primaryKeyLivenessInfo().isLive(nowInSec)) + return true; + + for (Cell cell : this) + if (cell.isLive(nowInSec)) + return true; + + return false; + } + + public boolean isEmpty() + { + return !primaryKeyLivenessInfo().hasTimestamp() + && deletion().isLive() + && !iterator().hasNext() + && !hasComplexDeletion(); + } + + public boolean isStatic() + { + return clustering() == Clustering.STATIC_CLUSTERING; + } + + public void digest(MessageDigest digest) + { + FBUtilities.updateWithByte(digest, kind().ordinal()); + clustering().digest(digest); + + deletion().digest(digest); + primaryKeyLivenessInfo().digest(digest); + + Iterator iter = columns().complexColumns(); + while (iter.hasNext()) + getDeletion(iter.next()).digest(digest); + + for (Cell cell : this) + cell.digest(digest); + } + + /** + * Copy this row to the provided writer. + * + * @param writer the row writer to write this row to. + */ + public void copyTo(Row.Writer writer) + { + Rows.writeClustering(clustering(), writer); + writer.writePartitionKeyLivenessInfo(primaryKeyLivenessInfo()); + writer.writeRowDeletion(deletion()); + + for (Cell cell : this) + cell.writeTo(writer); + + for (int i = 0; i < columns().complexColumnCount(); i++) + { + ColumnDefinition c = columns().getComplex(i); + DeletionTime dt = getDeletion(c); + if (!dt.isLive()) + writer.writeComplexDeletion(c, dt); + } + writer.endOfRow(); + } + + public void validateData(CFMetaData metadata) + { + Clustering clustering = clustering(); + for (int i = 0; i < clustering.size(); i++) + { + ByteBuffer value = clustering.get(i); + if (value != null) + metadata.comparator.subtype(i).validate(value); + } + + primaryKeyLivenessInfo().validate(); + if (deletion().localDeletionTime() < 0) + throw new MarshalException("A local deletion time should not be negative"); + + for (Cell cell : this) + cell.validate(); + } + + public String toString(CFMetaData metadata) + { + return toString(metadata, false); + } + + public String toString(CFMetaData metadata, boolean fullDetails) + { + StringBuilder sb = new StringBuilder(); + sb.append("Row"); + if (fullDetails) + { + sb.append("[info=").append(primaryKeyLivenessInfo()); + if (!deletion().isLive()) + sb.append(" del=").append(deletion()); + sb.append(" ]"); + } + sb.append(": ").append(clustering().toString(metadata)).append(" | "); + boolean isFirst = true; + ColumnDefinition prevColumn = null; + for (Cell cell : this) + { + if (isFirst) isFirst = false; else sb.append(", "); + if (fullDetails) + { + if (cell.column().isComplex() && !cell.column().equals(prevColumn)) + { + DeletionTime complexDel = getDeletion(cell.column()); + if (!complexDel.isLive()) + sb.append("del(").append(cell.column().name).append(")=").append(complexDel).append(", "); + } + sb.append(cell); + prevColumn = cell.column(); + } + else + { + sb.append(cell.column().name); + if (cell.column().type instanceof CollectionType) + { + CollectionType ct = (CollectionType)cell.column().type; + sb.append("[").append(ct.nameComparator().getString(cell.path().get(0))).append("]"); + sb.append("=").append(ct.valueComparator().getString(cell.value())); + } + else + { + sb.append("=").append(cell.column().type.getString(cell.value())); + } + } + } + return sb.toString(); + } + + @Override + public boolean equals(Object other) + { + if(!(other instanceof Row)) + return false; + + Row that = (Row)other; + if (!this.clustering().equals(that.clustering()) + || !this.columns().equals(that.columns()) + || !this.primaryKeyLivenessInfo().equals(that.primaryKeyLivenessInfo()) + || !this.deletion().equals(that.deletion())) + return false; + + Iterator thisCells = this.iterator(); + Iterator thatCells = that.iterator(); + while (thisCells.hasNext()) + { + if (!thatCells.hasNext() || !thisCells.next().equals(thatCells.next())) + return false; + } + return !thatCells.hasNext(); + } + + @Override + public int hashCode() + { + int hash = Objects.hash(clustering(), columns(), primaryKeyLivenessInfo(), deletion()); + for (Cell cell : this) + hash += 31 * cell.hashCode(); + return hash; + } +} diff --git a/src/java/org/apache/cassandra/db/rows/AbstractUnfilteredRowIterator.java b/src/java/org/apache/cassandra/db/rows/AbstractUnfilteredRowIterator.java new file mode 100644 index 0000000000..5bfd1a384b --- /dev/null +++ b/src/java/org/apache/cassandra/db/rows/AbstractUnfilteredRowIterator.java @@ -0,0 +1,107 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.cassandra.db.rows; + +import java.util.Objects; + +import com.google.common.collect.AbstractIterator; +import com.google.common.collect.Iterators; + +import org.apache.cassandra.config.CFMetaData; +import org.apache.cassandra.db.*; + +public abstract class AbstractUnfilteredRowIterator extends AbstractIterator implements UnfilteredRowIterator +{ + protected final CFMetaData metadata; + protected final DecoratedKey partitionKey; + protected final DeletionTime partitionLevelDeletion; + protected final PartitionColumns columns; + protected final Row staticRow; + protected final boolean isReverseOrder; + protected final RowStats stats; + + protected AbstractUnfilteredRowIterator(CFMetaData metadata, + DecoratedKey partitionKey, + DeletionTime partitionLevelDeletion, + PartitionColumns columns, + Row staticRow, + boolean isReverseOrder, + RowStats stats) + { + this.metadata = metadata; + this.partitionKey = partitionKey; + this.partitionLevelDeletion = partitionLevelDeletion; + this.columns = columns; + this.staticRow = staticRow; + this.isReverseOrder = isReverseOrder; + this.stats = stats; + } + + public CFMetaData metadata() + { + return metadata; + } + + public PartitionColumns columns() + { + return columns; + } + + public boolean isReverseOrder() + { + return isReverseOrder; + } + + public DecoratedKey partitionKey() + { + return partitionKey; + } + + public DeletionTime partitionLevelDeletion() + { + return partitionLevelDeletion; + } + + public Row staticRow() + { + return staticRow; + } + + public RowStats stats() + { + return stats; + } + + public void close() + { + } + + public static boolean equal(UnfilteredRowIterator a, UnfilteredRowIterator b) + { + return Objects.equals(a.columns(), b.columns()) + && Objects.equals(a.metadata(), b.metadata()) + && Objects.equals(a.isReverseOrder(), b.isReverseOrder()) + && Objects.equals(a.partitionKey(), b.partitionKey()) + && Objects.equals(a.partitionLevelDeletion(), b.partitionLevelDeletion()) + && Objects.equals(a.staticRow(), b.staticRow()) + && Objects.equals(a.stats(), b.stats()) + && Objects.equals(a.metadata(), b.metadata()) + && Iterators.elementsEqual(a, b); + } + +} diff --git a/src/java/org/apache/cassandra/db/rows/Cell.java b/src/java/org/apache/cassandra/db/rows/Cell.java new file mode 100644 index 0000000000..80bf9019f7 --- /dev/null +++ b/src/java/org/apache/cassandra/db/rows/Cell.java @@ -0,0 +1,142 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.cassandra.db.rows; + +import java.nio.ByteBuffer; +import java.security.MessageDigest; + +import org.apache.cassandra.config.ColumnDefinition; +import org.apache.cassandra.db.Aliasable; +import org.apache.cassandra.db.LivenessInfo; + +/** + * A cell holds a single "simple" value for a given column, as well as "liveness" + * informations regarding that value. + *

+ * The is 2 kind of columns: simple ones and complex ones. + * Simple columns have only a single associated cell, while complex ones, + * the one corresponding to non-frozen collections and UDTs, are comprised + * of multiple cells. For complex columns, the different cells are distinguished + * by their cell path. + *

+ * We can also distinguish different kind of cells based on the property of their + * {@link #livenessInfo}: + * 1) "Normal" cells: their liveness info has no ttl and no deletion time. + * 2) Expiring cells: their liveness info has both a ttl and a deletion time (the latter + * deciding when the cell is actually expired). + * 3) Tombstones/deleted cells: their liveness info has a deletion time but no ttl. Those + * cells don't really have a value but their {@link #value} method return an empty + * buffer by convention. + */ +public interface Cell extends Aliasable +{ + /** + * The column this cell belongs to. + * + * @return the column this cell belongs to. + */ + public ColumnDefinition column(); + + /** + * Whether the cell is a counter cell or not. + * + * @return whether the cell is a counter cell or not. + */ + public boolean isCounterCell(); + + /** + * The cell value. + * + * @return the cell value. + */ + public ByteBuffer value(); + + /** + * The liveness info of the cell, that is its timestamp and whether it is + * expiring, deleted or none of the above. + * + * @return the cell {@link LivenessInfo}. + */ + public LivenessInfo livenessInfo(); + + /** + * Whether the cell is a tombstone or not. + * + * @return whether the cell is a tombstone or not. + */ + public boolean isTombstone(); + + /** + * Whether the cell is an expiring one or not. + *

+ * Note that this only correspond to whether the cell liveness info + * have a TTL or not, but doesn't tells whether the cell is already expired + * or not. You should use {@link #isLive} for that latter information. + * + * @return whether the cell is an expiring one or not. + */ + public boolean isExpiring(); + + /** + * Whether the cell is live or not given the current time. + * + * @param nowInSec the current time in seconds. This is used to + * decide if an expiring cell is expired or live. + * @return whether the cell is live or not at {@code nowInSec}. + */ + public boolean isLive(int nowInSec); + + /** + * For cells belonging to complex types (non-frozen collection and UDT), the + * path to the cell. + * + * @return the cell path for cells of complex column, and {@code null} for other cells. + */ + public CellPath path(); + + /** + * Write the cell to the provided writer. + * + * @param writer the row writer to write the cell to. + */ + public void writeTo(Row.Writer writer); + + /** + * Adds the cell to the provided digest. + * + * @param digest the {@code MessageDigest} to add the cell to. + */ + public void digest(MessageDigest digest); + + /** + * Validate the cell value. + * + * @throws MarshalException if the cell value is not a valid value for + * the column type this is a cell of. + */ + public void validate(); + + /** + * The size of the data hold by this cell. + * + * This is mainly used to verify if batches goes over a given size. + * + * @return the size used by the data of this cell. + */ + public int dataSize(); +} diff --git a/src/java/org/apache/cassandra/db/rows/CellData.java b/src/java/org/apache/cassandra/db/rows/CellData.java new file mode 100644 index 0000000000..29eac0115f --- /dev/null +++ b/src/java/org/apache/cassandra/db/rows/CellData.java @@ -0,0 +1,275 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.cassandra.db.rows; + +import java.nio.ByteBuffer; +import java.util.Arrays; + +import org.apache.cassandra.config.ColumnDefinition; +import org.apache.cassandra.db.*; +import org.apache.cassandra.utils.ObjectSizes; + +/** + * Contains (non-counter) cell data for one or more rows. + */ +class CellData +{ + private boolean isCounter; + + private ByteBuffer[] values; + private final LivenessInfoArray livenessInfos; + + CellData(int initialCellCapacity, boolean isCounter) + { + this.isCounter = isCounter; + this.values = new ByteBuffer[initialCellCapacity]; + this.livenessInfos = new LivenessInfoArray(initialCellCapacity); + } + + public void setCell(int idx, ByteBuffer value, LivenessInfo info) + { + ensureCapacity(idx); + values[idx] = value; + livenessInfos.set(idx, info); + } + + public boolean hasCell(int idx) + { + return idx < values.length && values[idx] != null; + } + + public ByteBuffer value(int idx) + { + return values[idx]; + } + + public void setValue(int idx, ByteBuffer value) + { + values[idx] = value; + } + + private void ensureCapacity(int idxToSet) + { + int originalCapacity = values.length; + if (idxToSet < originalCapacity) + return; + + int newCapacity = RowDataBlock.computeNewCapacity(originalCapacity, idxToSet); + + values = Arrays.copyOf(values, newCapacity); + livenessInfos.resize(newCapacity); + } + + // Swap cell i and j + public void swapCell(int i, int j) + { + ensureCapacity(Math.max(i, j)); + + ByteBuffer value = values[j]; + values[j] = values[i]; + values[i] = value; + + livenessInfos.swap(i, j); + } + + // Merge cell i into j + public void mergeCell(int i, int j, int nowInSec) + { + if (isCounter) + mergeCounterCell(this, i, this, j, this, j, nowInSec); + else + mergeRegularCell(this, i, this, j, this, j, nowInSec); + } + + private static boolean handleNoCellCase(CellData d1, int i1, CellData d2, int i2, CellData merged, int iMerged) + { + if (!d1.hasCell(i1)) + { + if (d2.hasCell(i2)) + d2.moveCell(i2, merged, iMerged); + return true; + } + if (!d2.hasCell(i2)) + { + d1.moveCell(i1, merged, iMerged); + return true; + } + return false; + } + + public static void mergeRegularCell(CellData d1, int i1, CellData d2, int i2, CellData merged, int iMerged, int nowInSec) + { + if (handleNoCellCase(d1, i1, d2, i2, merged, iMerged)) + return; + + Conflicts.Resolution res = Conflicts.resolveRegular(d1.livenessInfos.timestamp(i1), + d1.livenessInfos.isLive(i1, nowInSec), + d1.livenessInfos.localDeletionTime(i1), + d1.values[i1], + d2.livenessInfos.timestamp(i2), + d2.livenessInfos.isLive(i2, nowInSec), + d2.livenessInfos.localDeletionTime(i2), + d2.values[i2]); + + assert res != Conflicts.Resolution.MERGE; + if (res == Conflicts.Resolution.LEFT_WINS) + d1.moveCell(i1, merged, iMerged); + else + d2.moveCell(i2, merged, iMerged); + } + + public static void mergeCounterCell(CellData d1, int i1, CellData d2, int i2, CellData merged, int iMerged, int nowInSec) + { + if (handleNoCellCase(d1, i1, d2, i2, merged, iMerged)) + return; + + Conflicts.Resolution res = Conflicts.resolveCounter(d1.livenessInfos.timestamp(i1), + d1.livenessInfos.isLive(i1, nowInSec), + d1.values[i1], + d2.livenessInfos.timestamp(i2), + d2.livenessInfos.isLive(i2, nowInSec), + d2.values[i2]); + + switch (res) + { + case LEFT_WINS: + d1.moveCell(i1, merged, iMerged); + break; + case RIGHT_WINS: + d2.moveCell(i2, merged, iMerged); + break; + default: + merged.values[iMerged] = Conflicts.mergeCounterValues(d1.values[i1], d2.values[i2]); + if (d1.livenessInfos.timestamp(i1) > d2.livenessInfos.timestamp(i2)) + merged.livenessInfos.set(iMerged, d1.livenessInfos.timestamp(i1), d1.livenessInfos.ttl(i1), d1.livenessInfos.localDeletionTime(i1)); + else + merged.livenessInfos.set(iMerged, d2.livenessInfos.timestamp(i2), d2.livenessInfos.ttl(i2), d2.livenessInfos.localDeletionTime(i2)); + break; + } + } + + // Move cell i into j + public void moveCell(int i, int j) + { + moveCell(i, this, j); + } + + public void moveCell(int i, CellData target, int j) + { + if (!hasCell(i) || (target == this && i == j)) + return; + + target.ensureCapacity(j); + + target.values[j] = values[i]; + target.livenessInfos.set(j, livenessInfos.timestamp(i), + livenessInfos.ttl(i), + livenessInfos.localDeletionTime(i)); + } + + public int dataSize() + { + int size = livenessInfos.dataSize(); + for (int i = 0; i < values.length; i++) + if (values[i] != null) + size += values[i].remaining(); + return size; + } + + public void clear() + { + Arrays.fill(values, null); + livenessInfos.clear(); + } + + public long unsharedHeapSizeExcludingData() + { + return ObjectSizes.sizeOnHeapExcludingData(values) + + livenessInfos.unsharedHeapSize(); + } + + @Override + public String toString() + { + StringBuilder sb = new StringBuilder(); + sb.append("CellData(size=").append(values.length); + if (isCounter) + sb.append(", counter"); + sb.append("){"); + LivenessInfoArray.Cursor cursor = LivenessInfoArray.newCursor(); + for (int i = 0; i < values.length; i++) + { + if (values[i] == null) + { + sb.append("[null]"); + continue; + } + sb.append("[len(v)=").append(values[i].remaining()); + sb.append(", info=").append(cursor.setTo(livenessInfos, i)); + sb.append("]"); + } + return sb.append("}").toString(); + } + + static class ReusableCell extends AbstractCell + { + private final LivenessInfoArray.Cursor cursor = LivenessInfoArray.newCursor(); + + private CellData data; + private ColumnDefinition column; + protected int idx; + + ReusableCell setTo(CellData data, ColumnDefinition column, int idx) + { + if (!data.hasCell(idx)) + return null; + + this.data = data; + this.column = column; + this.idx = idx; + + cursor.setTo(data.livenessInfos, idx); + return this; + } + + public ColumnDefinition column() + { + return column; + } + + public boolean isCounterCell() + { + return data.isCounter && !cursor.hasLocalDeletionTime(); + } + + public ByteBuffer value() + { + return data.value(idx); + } + + public LivenessInfo livenessInfo() + { + return cursor; + } + + public CellPath path() + { + return null; + } + } +} diff --git a/src/java/org/apache/cassandra/db/rows/CellPath.java b/src/java/org/apache/cassandra/db/rows/CellPath.java new file mode 100644 index 0000000000..8233ac27b2 --- /dev/null +++ b/src/java/org/apache/cassandra/db/rows/CellPath.java @@ -0,0 +1,127 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.cassandra.db.rows; + +import java.io.DataInput; +import java.io.IOException; +import java.nio.ByteBuffer; +import java.security.MessageDigest; +import java.util.Objects; + +import org.apache.cassandra.db.TypeSizes; +import org.apache.cassandra.io.util.DataOutputPlus; + +/** + * A path for a cell belonging to a complex column type (non-frozen collection or UDT). + */ +public abstract class CellPath +{ + public static final CellPath BOTTOM = new EmptyCellPath(); + public static final CellPath TOP = new EmptyCellPath(); + + public abstract int size(); + public abstract ByteBuffer get(int i); + + // The only complex we currently have are collections that have only one value. + public static CellPath create(ByteBuffer value) + { + assert value != null; + return new SimpleCellPath(new ByteBuffer[]{ value }); + } + + public int dataSize() + { + int size = 0; + for (int i = 0; i < size(); i++) + size += get(i).remaining(); + return size; + } + + public void digest(MessageDigest digest) + { + for (int i = 0; i < size(); i++) + digest.update(get(i).duplicate()); + } + + @Override + public final int hashCode() + { + int result = 31; + for (int i = 0; i < size(); i++) + result += 31 * Objects.hash(get(i)); + return result; + } + + @Override + public final boolean equals(Object o) + { + if(!(o instanceof CellPath)) + return false; + + CellPath that = (CellPath)o; + if (this.size() != that.size()) + return false; + + for (int i = 0; i < size(); i++) + if (!Objects.equals(this.get(i), that.get(i))) + return false; + + return true; + } + + public interface Serializer + { + public void serialize(CellPath path, DataOutputPlus out) throws IOException; + public CellPath deserialize(DataInput in) throws IOException; + public long serializedSize(CellPath path, TypeSizes sizes); + public void skip(DataInput in) throws IOException; + } + + static class SimpleCellPath extends CellPath + { + protected final ByteBuffer[] values; + + public SimpleCellPath(ByteBuffer[] values) + { + this.values = values; + } + + public int size() + { + return values.length; + } + + public ByteBuffer get(int i) + { + return values[i]; + } + } + + private static class EmptyCellPath extends CellPath + { + public int size() + { + return 0; + } + + public ByteBuffer get(int i) + { + throw new UnsupportedOperationException(); + } + } +} diff --git a/src/java/org/apache/cassandra/db/rows/Cells.java b/src/java/org/apache/cassandra/db/rows/Cells.java new file mode 100644 index 0000000000..1e329e59aa --- /dev/null +++ b/src/java/org/apache/cassandra/db/rows/Cells.java @@ -0,0 +1,371 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.cassandra.db.rows; + +import java.nio.ByteBuffer; +import java.util.Comparator; +import java.util.Iterator; + +import org.apache.cassandra.config.ColumnDefinition; +import org.apache.cassandra.db.*; +import org.apache.cassandra.db.context.CounterContext; +import org.apache.cassandra.db.index.SecondaryIndexManager; +import org.apache.cassandra.utils.ByteBufferUtil; + +/** + * Static methods to work on cells. + */ +public abstract class Cells +{ + private Cells() {} + + /** + * Writes a tombstone cell to the provided writer. + * + * @param writer the {@code Row.Writer} to write the tombstone to. + * @param column the column for the tombstone. + * @param timestamp the timestamp for the tombstone. + * @param localDeletionTime the local deletion time (in seconds) for the tombstone. + */ + public static void writeTombstone(Row.Writer writer, ColumnDefinition column, long timestamp, int localDeletionTime) + { + writer.writeCell(column, false, ByteBufferUtil.EMPTY_BYTE_BUFFER, SimpleLivenessInfo.forDeletion(timestamp, localDeletionTime), null); + } + + /** + * Computes the difference between a cell and the result of merging this + * cell to other cells. + *

+ * This method is used when cells from multiple sources are merged and we want to + * find for a given source if it was up to date for that cell, and if not, what + * should be sent to the source to repair it. + * + * @param merged the cell that is the result of merging multiple source. + * @param cell the cell from one of the source that has been merged to yied + * {@code merged}. + * @return {@code null} if the source having {@code cell} is up-to-date for that + * cell, or a cell that applied to the source will "repair" said source otherwise. + */ + public static Cell diff(Cell merged, Cell cell) + { + // Note that it's enough to check if merged is a counterCell. If it isn't and + // cell is one, it means that merged is a tombstone with a greater timestamp + // than cell, because that's the only case where reconciling a counter with + // a tombstone don't yield a counter. If that's the case, the normal path will + // return what it should. + if (merged.isCounterCell()) + { + if (merged.livenessInfo().supersedes(cell.livenessInfo())) + return merged; + + // Reconciliation never returns something with a timestamp strictly lower than its operand. This + // means we're in the case where merged.timestamp() == cell.timestamp(). As 1) tombstones + // always win over counters (CASSANDRA-7346) and 2) merged is a counter, it follows that cell + // can't be a tombstone or merged would be one too. + assert !cell.isTombstone(); + + CounterContext.Relationship rel = CounterContext.instance().diff(merged.value(), cell.value()); + return (rel == CounterContext.Relationship.GREATER_THAN || rel == CounterContext.Relationship.DISJOINT) ? merged : null; + } + return merged.livenessInfo().supersedes(cell.livenessInfo()) ? merged : null; + } + + /** + * Reconciles/merges two cells, one being an update to an existing cell, + * yielding index updates if appropriate. + *

+ * Note that this method assumes that the provided cells can meaningfully + * be reconciled together, that is that those cells are for the same row and same + * column (and same cell path if the column is complex). + *

+ * Also note that which cell is provided as {@code existing} and which is + * provided as {@code update} matters for index updates. + * + * @param clustering the clustering for the row the cells to merge originate from. + * This is only used for index updates, so this can be {@code null} if + * {@code indexUpdater == SecondaryIndexManager.nullUpdater}. + * @param existing the pre-existing cell, the one that is updated. This can be + * {@code null} if this reconciliation correspond to an insertion. + * @param update the newly added cell, the update. This can be {@code null} out + * of convenience, in which case this function simply copy {@code existing} to + * {@code writer}. + * @param deletion the deletion time that applies to the cells being considered. + * This deletion time may delete both {@code existing} or {@code update}. + * @param writer the row writer to which the result of the reconciliation is written. + * @param nowInSec the current time in seconds (which plays a role during reconciliation + * because deleted cells always have precedence on timestamp equality and deciding if a + * cell is a live or not depends on the current time due to expiring cells). + * @param indexUpdater an index updater to which the result of the reconciliation is + * signaled (if relevant, that is if the update is not simply ignored by the reconciliation). + * This cannot be {@code null} but {@code SecondaryIndexManager.nullUpdater} can be passed. + * + * @return the timestamp delta between existing and update, or {@code Long.MAX_VALUE} if one + * of them is {@code null} or deleted by {@code deletion}). + */ + public static long reconcile(Clustering clustering, + Cell existing, + Cell update, + DeletionTime deletion, + Row.Writer writer, + int nowInSec, + SecondaryIndexManager.Updater indexUpdater) + { + existing = existing == null || deletion.deletes(existing.livenessInfo()) ? null : existing; + update = update == null || deletion.deletes(update.livenessInfo()) ? null : update; + if (existing == null || update == null) + { + if (update != null) + { + // It's inefficient that we call maybeIndex (which is for primary key indexes) on every cell, but + // we'll need to fix that damn 2ndary index API to avoid that. + updatePKIndexes(clustering, update, nowInSec, indexUpdater); + indexUpdater.insert(clustering, update); + update.writeTo(writer); + } + else if (existing != null) + { + existing.writeTo(writer); + } + return Long.MAX_VALUE; + } + + Cell reconciled = reconcile(existing, update, nowInSec); + reconciled.writeTo(writer); + + // Note that this test rely on reconcile returning either 'existing' or 'update'. That's not true for counters but we don't index them + if (reconciled == update) + { + updatePKIndexes(clustering, update, nowInSec, indexUpdater); + indexUpdater.update(clustering, existing, reconciled); + } + return Math.abs(existing.livenessInfo().timestamp() - update.livenessInfo().timestamp()); + } + + private static void updatePKIndexes(Clustering clustering, Cell cell, int nowInSec, SecondaryIndexManager.Updater indexUpdater) + { + if (indexUpdater != SecondaryIndexManager.nullUpdater && cell.isLive(nowInSec)) + indexUpdater.maybeIndex(clustering, cell.livenessInfo().timestamp(), cell.livenessInfo().ttl(), DeletionTime.LIVE); + } + + /** + * Reconciles/merge two cells. + *

+ * Note that this method assumes that the provided cells can meaningfully + * be reconciled together, that is that cell are for the same row and same + * column (and same cell path if the column is complex). + *

+ * This method is commutative over it's cells arguments: {@code reconcile(a, b, n) == reconcile(b, a, n)}. + * + * @param c1 the first cell participating in the reconciliation. + * @param c2 the second cell participating in the reconciliation. + * @param nowInSec the current time in seconds (which plays a role during reconciliation + * because deleted cells always have precedence on timestamp equality and deciding if a + * cell is a live or not depends on the current time due to expiring cells). + * + * @return a cell corresponding to the reconciliation of {@code c1} and {@code c2}. + * For non-counter cells, this will always be either {@code c1} or {@code c2}, but for + * counter cells this can be a newly allocated cell. + */ + public static Cell reconcile(Cell c1, Cell c2, int nowInSec) + { + if (c1 == null) + return c2 == null ? null : c2; + if (c2 == null) + return c1; + + if (c1.isCounterCell() || c2.isCounterCell()) + { + Conflicts.Resolution res = Conflicts.resolveCounter(c1.livenessInfo().timestamp(), + c1.isLive(nowInSec), + c1.value(), + c2.livenessInfo().timestamp(), + c2.isLive(nowInSec), + c2.value()); + + switch (res) + { + case LEFT_WINS: return c1; + case RIGHT_WINS: return c2; + default: + ByteBuffer merged = Conflicts.mergeCounterValues(c1.value(), c2.value()); + LivenessInfo mergedInfo = c1.livenessInfo().mergeWith(c2.livenessInfo()); + + // We save allocating a new cell object if it turns out that one cell was + // a complete superset of the other + if (merged == c1.value() && mergedInfo == c1.livenessInfo()) + return c1; + else if (merged == c2.value() && mergedInfo == c2.livenessInfo()) + return c2; + else // merge clocks and timestamps. + return create(c1.column(), true, merged, mergedInfo, null); + } + } + + Conflicts.Resolution res = Conflicts.resolveRegular(c1.livenessInfo().timestamp(), + c1.isLive(nowInSec), + c1.livenessInfo().localDeletionTime(), + c1.value(), + c2.livenessInfo().timestamp(), + c2.isLive(nowInSec), + c2.livenessInfo().localDeletionTime(), + c2.value()); + assert res != Conflicts.Resolution.MERGE; + return res == Conflicts.Resolution.LEFT_WINS ? c1 : c2; + } + + /** + * Computes the reconciliation of a complex column given its pre-existing + * cells and the ones it is updated with, and generating index update if + * appropriate. + *

+ * Note that this method assumes that the provided cells can meaningfully + * be reconciled together, that is that the cells are for the same row and same + * complex column. + *

+ * Also note that which cells is provided as {@code existing} and which are + * provided as {@code update} matters for index updates. + * + * @param clustering the clustering for the row the cells to merge originate from. + * This is only used for index updates, so this can be {@code null} if + * {@code indexUpdater == SecondaryIndexManager.nullUpdater}. + * @param column the complex column the cells are for. + * @param existing the pre-existing cells, the ones that are updated. This can be + * {@code null} if this reconciliation correspond to an insertion. + * @param update the newly added cells, the update. This can be {@code null} out + * of convenience, in which case this function simply copy the cells from + * {@code existing} to {@code writer}. + * @param deletion the deletion time that applies to the cells being considered. + * This deletion time may delete cells in both {@code existing} and {@code update}. + * @param writer the row writer to which the result of the reconciliation is written. + * @param nowInSec the current time in seconds (which plays a role during reconciliation + * because deleted cells always have precedence on timestamp equality and deciding if a + * cell is a live or not depends on the current time due to expiring cells). + * @param indexUpdater an index updater to which the result of the reconciliation is + * signaled (if relevant, that is if the updates are not simply ignored by the reconciliation). + * This cannot be {@code null} but {@code SecondaryIndexManager.nullUpdater} can be passed. + * + * @return the smallest timestamp delta between corresponding cells from existing and update. A + * timestamp delta being computed as the difference between a cell from {@code update} and the + * cell in {@code existing} having the same cell path (if such cell exists). If the intersection + * of cells from {@code existing} and {@code update} having the same cell path is empty, this + * returns {@code Long.MAX_VALUE}. + */ + public static long reconcileComplex(Clustering clustering, + ColumnDefinition column, + Iterator existing, + Iterator update, + DeletionTime deletion, + Row.Writer writer, + int nowInSec, + SecondaryIndexManager.Updater indexUpdater) + { + Comparator comparator = column.cellPathComparator(); + Cell nextExisting = getNext(existing); + Cell nextUpdate = getNext(update); + long timeDelta = Long.MAX_VALUE; + while (nextExisting != null || nextUpdate != null) + { + int cmp = nextExisting == null ? 1 + : (nextUpdate == null ? -1 + : comparator.compare(nextExisting.path(), nextUpdate.path())); + if (cmp < 0) + { + reconcile(clustering, nextExisting, null, deletion, writer, nowInSec, indexUpdater); + nextExisting = getNext(existing); + } + else if (cmp > 0) + { + reconcile(clustering, null, nextUpdate, deletion, writer, nowInSec, indexUpdater); + nextUpdate = getNext(update); + } + else + { + timeDelta = Math.min(timeDelta, reconcile(clustering, nextExisting, nextUpdate, deletion, writer, nowInSec, indexUpdater)); + nextExisting = getNext(existing); + nextUpdate = getNext(update); + } + } + return timeDelta; + } + + private static Cell getNext(Iterator iterator) + { + return iterator == null || !iterator.hasNext() ? null : iterator.next(); + } + + /** + * Creates a simple cell. + *

+ * Note that in general cell objects are created by the container they are in and so this method should + * only be used in a handful of cases when we know it's the right thing to do. + * + * @param column the column for the cell to create. + * @param isCounter whether the create cell should be a counter one. + * @param value the value for the cell. + * @param info the liveness info for the cell. + * @param path the cell path for the cell. + * @return the newly allocated cell object. + */ + public static Cell create(ColumnDefinition column, boolean isCounter, ByteBuffer value, LivenessInfo info, CellPath path) + { + return new SimpleCell(column, isCounter, value, info, path); + } + + private static class SimpleCell extends AbstractCell + { + private final ColumnDefinition column; + private final boolean isCounter; + private final ByteBuffer value; + private final LivenessInfo info; + private final CellPath path; + + private SimpleCell(ColumnDefinition column, boolean isCounter, ByteBuffer value, LivenessInfo info, CellPath path) + { + this.column = column; + this.isCounter = isCounter; + this.value = value; + this.info = info.takeAlias(); + this.path = path; + } + + public ColumnDefinition column() + { + return column; + } + + public boolean isCounterCell() + { + return isCounter; + } + + public ByteBuffer value() + { + return value; + } + + public LivenessInfo livenessInfo() + { + return info; + } + + public CellPath path() + { + return path; + } + } +} diff --git a/src/java/org/apache/cassandra/db/rows/ColumnData.java b/src/java/org/apache/cassandra/db/rows/ColumnData.java new file mode 100644 index 0000000000..ea472eb13a --- /dev/null +++ b/src/java/org/apache/cassandra/db/rows/ColumnData.java @@ -0,0 +1,61 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.cassandra.db.rows; + +import java.util.Iterator; + +import org.apache.cassandra.config.ColumnDefinition; +import org.apache.cassandra.db.DeletionTime; + +public class ColumnData +{ + private final ColumnDefinition column; + private final Cell cell; + private final Iterator cells; + private final DeletionTime complexDeletion; + + ColumnData(ColumnDefinition column, Cell cell, Iterator cells, DeletionTime complexDeletion) + { + assert column != null && (cell != null || (column.isComplex() && cells != null && complexDeletion != null)); + + this.column = column; + this.cell = cell; + this.cells = cells; + this.complexDeletion = complexDeletion; + } + + public ColumnDefinition column() + { + return column; + } + + public Cell cell() + { + return cell; + } + + public Iterator cells() + { + return cells; + } + + public DeletionTime complexDeletion() + { + return complexDeletion; + } +} diff --git a/src/java/org/apache/cassandra/db/rows/ComplexRowDataBlock.java b/src/java/org/apache/cassandra/db/rows/ComplexRowDataBlock.java new file mode 100644 index 0000000000..75df8743df --- /dev/null +++ b/src/java/org/apache/cassandra/db/rows/ComplexRowDataBlock.java @@ -0,0 +1,796 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.cassandra.db.rows; + +import java.nio.ByteBuffer; +import java.util.*; + +import com.google.common.collect.UnmodifiableIterator; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import org.apache.cassandra.config.ColumnDefinition; +import org.apache.cassandra.db.*; +import org.apache.cassandra.utils.ObjectSizes; + +/** + * Holds cells data and complex deletions for the complex columns of one or more rows. + *

+ * Contrarily to {@code SimpleRowDataBlock}, each complex column can have multiple cells and + * we thus can't use a similar dense encoding. Instead, we still store the actual cell data + * in a {@code CellData} object, but we add a level of indirection (the cellIdx array in + * {@link ComplexCellBlock}) which for every column of every row stores 2 indexes: the index + * in the {@code CellData} where the first cell for this column is, and the the index of the + * last cell (or rather, the index to the first cell that does not belong to that column). + *

+ * What makes this a little bit more complicated however is that in some cases (for + * {@link PartitionUpdate} typically), we need to be able to swap rows inside a + * {@code ComplexRowDataBlock} and the extra level of indirection makes that more complex. + * So in practice, we have 2 separate sub-implementation of a {@code ComplexRowDataBlock}: + * - The first one, {@code SimpleComplexRowDataBlock} does not support swapping rows + * (and is thus only used when we don't need to) but it uses a single {@code CellData} + * for all the rows stored. + * - The second one, {@code SortableComplexRowDataBlock}, uses one separate {@code CellData} + * per row (in fact, a {@code ComplexCellBlock} which groups the cell data with the + * indexing array discussed above) and simply keeps those per-row block in a list. It + * is thus less compact in memory but make the swapping of rows trivial. + */ +public abstract class ComplexRowDataBlock +{ + private static final Logger logger = LoggerFactory.getLogger(ComplexRowDataBlock.class); + + private final Columns columns; + + // For each complex column, it's deletion time (if any): the nth complex column of row i + // will have it's deletion time at complexDelTimes[(i * ccs) + n] where ccs it the number + // of complex columns in 'columns'. + final DeletionTimeArray complexDelTimes; + + protected ComplexRowDataBlock(Columns columns, int rows) + { + this.columns = columns; + + int columnCount = rows * columns.complexColumnCount(); + this.complexDelTimes = new DeletionTimeArray(columnCount); + } + + public static ComplexRowDataBlock create(Columns columns, int rows, boolean sortable, boolean isCounter) + { + return sortable + ? new SortableComplexRowDataBlock(columns, rows, isCounter) + : new SimpleComplexRowDataBlock(columns, rows, isCounter); + } + + public Columns columns() + { + return columns; + } + + public CellData cellData(int row) + { + return cellBlock(row).data; + } + + public int cellIdx(int row, ColumnDefinition c, CellPath path) + { + ComplexCellBlock block = cellBlock(row); + if (block == null) + return -1; + + int base = cellBlockBase(row); + int i = base + 2 * columns.complexIdx(c, 0); + + int start = block.cellIdx[i]; + int end = block.cellIdx[i+1]; + + if (i >= block.cellIdx.length || end <= start) + return -1; + + return Arrays.binarySearch(block.complexPaths, start, end, path, c.cellPathComparator()); + } + + // The following methods abstract the fact that we have 2 sub-implementations: both + // implementation will use a ComplexCellBlock to store a row, but one will use one + // ComplexCellBlock per row, while the other will store all rows into the same block. + + // Returns the cell block for a given row. Can return null if the asked row has no data. + protected abstract ComplexCellBlock cellBlock(int row); + // Same as cellBlock(), but create the proper block if the row doesn't exists and return it. + protected abstract ComplexCellBlock cellBlockForWritting(int row); + // The index in the block returned by cellBlock()/cellBlockFroWriting() where the row starts. + protected abstract int cellBlockBase(int row); + + protected abstract void swapCells(int i, int j); + protected abstract void mergeCells(int i, int j, int nowInSec); + protected abstract void moveCells(int i, int j); + + protected abstract long cellDataUnsharedHeapSizeExcludingData(); + protected abstract int dataCellSize(); + protected abstract void clearCellData(); + + // Swap row i and j + public void swap(int i, int j) + { + swapCells(i, j); + + int s = columns.complexColumnCount(); + for (int k = 0; k < s; k++) + complexDelTimes.swap(i * s + k, j * s + k); + } + + // Merge row i into j + public void merge(int i, int j, int nowInSec) + { + assert i > j; + + mergeCells(i, j, nowInSec); + + int s = columns.complexColumnCount(); + if (i * s >= complexDelTimes.size()) + return; + + for (int k = 0; k < s; k++) + if (complexDelTimes.supersedes(i * s + k, j * s + k)) + complexDelTimes.move(i * s + k, j * s + k); + } + + // Move row i into j + public void move(int i, int j) + { + moveCells(i, j); + ensureDelTimesCapacity(Math.max(i, j)); + int s = columns.complexColumnCount(); + for (int k = 0; k < s; k++) + complexDelTimes.move(i * s + k, j * s + k); + } + + public long unsharedHeapSizeExcludingData() + { + return cellDataUnsharedHeapSizeExcludingData() + complexDelTimes.unsharedHeapSize(); + } + + public int dataSize() + { + return dataCellSize() + complexDelTimes.dataSize(); + } + + public CellWriter cellWriter(boolean inOrderCells) + { + return new CellWriter(inOrderCells); + } + + public int complexDeletionIdx(int row, ColumnDefinition column) + { + int baseIdx = columns.complexIdx(column, 0); + if (baseIdx < 0) + return -1; + + int idx = (row * columns.complexColumnCount()) + baseIdx; + return idx < complexDelTimes.size() ? idx : -1; + } + + public boolean hasComplexDeletion(int row) + { + int base = row * columns.complexColumnCount(); + for (int i = base; i < base + columns.complexColumnCount(); i++) + if (!complexDelTimes.isLive(i)) + return true; + return false; + } + + public ByteBuffer getValue(int row, ColumnDefinition column, CellPath path) + { + CellData data = cellData(row); + assert data != null; + int idx = cellIdx(row, column, path); + return data.value(idx); + } + + public void setValue(int row, ColumnDefinition column, CellPath path, ByteBuffer value) + { + CellData data = cellData(row); + assert data != null; + int idx = cellIdx(row, column, path); + data.setValue(idx, value); + } + + public static ReusableIterator reusableComplexCells() + { + return new ReusableIterator(); + } + + public static DeletionTimeArray.Cursor complexDeletionCursor() + { + return new DeletionTimeArray.Cursor(); + } + + public static ReusableIterator reusableIterator() + { + return new ReusableIterator(); + } + + public void clear() + { + clearCellData(); + complexDelTimes.clear(); + } + + private void ensureDelTimesCapacity(int rowToSet) + { + int originalCapacity = complexDelTimes.size() / columns.complexColumnCount(); + if (rowToSet < originalCapacity) + return; + + int newCapacity = RowDataBlock.computeNewCapacity(originalCapacity, rowToSet); + complexDelTimes.resize(newCapacity * columns.complexColumnCount()); + } + + /** + * Simple sub-implementation that doesn't support swapping/sorting rows. + * The cell data for every row is stored in the same contiguous {@code ComplexCellBloc} + * object. + */ + private static class SimpleComplexRowDataBlock extends ComplexRowDataBlock + { + private static final long EMPTY_SIZE = ObjectSizes.measure(new SimpleComplexRowDataBlock(Columns.NONE, 0, false)); + + private final ComplexCellBlock cells; + + private SimpleComplexRowDataBlock(Columns columns, int rows, boolean isCounter) + { + super(columns, rows); + this.cells = new ComplexCellBlock(columns, rows, isCounter); + } + + protected ComplexCellBlock cellBlock(int row) + { + return cells; + } + + protected ComplexCellBlock cellBlockForWritting(int row) + { + cells.ensureCapacity(row); + return cells; + } + + protected int cellBlockBase(int row) + { + return 2 * row * columns().complexColumnCount(); + } + + // Swap cells from row i and j + public void swapCells(int i, int j) + { + throw new UnsupportedOperationException(); + } + + // Merge cells from row i into j + public void mergeCells(int i, int j, int nowInSec) + { + throw new UnsupportedOperationException(); + } + + // Move cells from row i into j + public void moveCells(int i, int j) + { + throw new UnsupportedOperationException(); + } + + protected long cellDataUnsharedHeapSizeExcludingData() + { + return EMPTY_SIZE + cells.unsharedHeapSizeExcludingData(); + } + + protected int dataCellSize() + { + return cells.dataSize(); + } + + protected void clearCellData() + { + cells.clear(); + } + } + + /** + * Sub-implementation that support swapping/sorting rows. + * The data for each row is stored in a different {@code ComplexCellBlock} object, + * making swapping rows easy. + */ + private static class SortableComplexRowDataBlock extends ComplexRowDataBlock + { + private static final long EMPTY_SIZE = ObjectSizes.measure(new SortableComplexRowDataBlock(Columns.NONE, 0, false)); + + // The cell data for each row. + private final List cells; + private final boolean isCounter; + + private SortableComplexRowDataBlock(Columns columns, int rows, boolean isCounter) + { + super(columns, rows); + this.cells = new ArrayList<>(rows); + this.isCounter = isCounter; + } + + protected ComplexCellBlock cellBlockForWritting(int row) + { + if (row < cells.size()) + return cells.get(row); + + // Make sure the list of size 'row-1' before the insertion, adding nulls if necessary, + // so that we do are writing row 'row' + ensureCapacity(row-1); + + assert row == cells.size(); + ComplexCellBlock block = new ComplexCellBlock(columns(), 1, isCounter); + cells.add(block); + return block; + } + + private void ensureCapacity(int row) + { + while (row >= cells.size()) + cells.add(null); + } + + protected ComplexCellBlock cellBlock(int row) + { + return row >= cells.size() ? null : cells.get(row); + } + + protected int cellBlockBase(int row) + { + return 0; + } + + // Swap row i and j + protected void swapCells(int i, int j) + { + int max = Math.max(i, j); + if (max >= cells.size()) + ensureCapacity(max); + + ComplexCellBlock block = cells.get(j); + move(i, j); + cells.set(i, block); + } + + // Merge row i into j + protected void mergeCells(int i, int j, int nowInSec) + { + assert i > j; + if (i >= cells.size()) + return; + + ComplexCellBlock b1 = cells.get(i); + if (b1 == null) + return; // nothing to merge into j + + ComplexCellBlock b2 = cells.get(j); + if (b2 == null) + { + cells.set(j, b1); + return; + } + + ComplexCellBlock merged = new ComplexCellBlock(columns(), 1, isCounter); + + int idxMerged = 0; + int s = columns().complexColumnCount(); + for (int k = 0; k < s; k++) + { + ColumnDefinition column = columns().getComplex(k); + Comparator comparator = column.cellPathComparator(); + + merged.cellIdx[2 * k] = idxMerged; + + int idx1 = b1.cellIdx[2 * k]; + int end1 = b1.cellIdx[2 * k + 1]; + int idx2 = b2.cellIdx[2 * k]; + int end2 = b2.cellIdx[2 * k + 1]; + + while (idx1 < end1 || idx2 < end2) + { + int cmp = idx1 >= end1 ? 1 + : (idx2 >= end2 ? -1 + : comparator.compare(b1.complexPaths[idx1], b2.complexPaths[idx2])); + + if (cmp == 0) + merge(b1, idx1++, b2, idx2++, merged, idxMerged++, nowInSec); + else if (cmp < 0) + copy(b1, idx1++, merged, idxMerged++); + else + copy(b2, idx2++, merged, idxMerged++); + } + + merged.cellIdx[2 * k + 1] = idxMerged; + } + + cells.set(j, merged); + } + + private void copy(ComplexCellBlock fromBlock, int fromIdx, ComplexCellBlock toBlock, int toIdx) + { + fromBlock.data.moveCell(fromIdx, toBlock.data, toIdx); + toBlock.ensureComplexPathsCapacity(toIdx); + toBlock.complexPaths[toIdx] = fromBlock.complexPaths[fromIdx]; + } + + private void merge(ComplexCellBlock b1, int idx1, ComplexCellBlock b2, int idx2, ComplexCellBlock mergedBlock, int mergedIdx, int nowInSec) + { + if (isCounter) + CellData.mergeCounterCell(b1.data, idx1, b2.data, idx2, mergedBlock.data, mergedIdx, nowInSec); + else + CellData.mergeRegularCell(b1.data, idx1, b2.data, idx2, mergedBlock.data, mergedIdx, nowInSec); + mergedBlock.ensureComplexPathsCapacity(mergedIdx); + mergedBlock.complexPaths[mergedIdx] = b1.complexPaths[idx1]; + } + + // Move row i into j + protected void moveCells(int i, int j) + { + int max = Math.max(i, j); + if (max >= cells.size()) + ensureCapacity(max); + + cells.set(j, cells.get(i)); + } + + protected long cellDataUnsharedHeapSizeExcludingData() + { + long size = EMPTY_SIZE; + for (ComplexCellBlock block : cells) + if (block != null) + size += block.unsharedHeapSizeExcludingData(); + return size; + } + + protected int dataCellSize() + { + int size = 0; + for (ComplexCellBlock block : cells) + if (block != null) + size += block.dataSize(); + return size; + } + + protected void clearCellData() + { + for (ComplexCellBlock block : cells) + if (block != null) + block.clear(); + } + } + + /** + * Stores complex column cell data for one or more rows. + *

+ * On top of a {@code CellData} object, this stores an index to where the cells + * of a given column start and stop in that {@code CellData} object (cellIdx) + * as well as the cell path for the cells (since {@code CellData} doesn't have those). + */ + private static class ComplexCellBlock + { + private final Columns columns; + + /* + * For a given complex column c, we have to store an unknown number of + * cells. So for each column of each row, we keep pointers (in data) + * to the start and end of the cells for this column (cells for a given + * columns are thus stored contiguously). + * For instance, if columns has 'c' complex columns, the x-th column of + * row 'n' will have it's cells in data at indexes + * [cellIdx[2 * (n * c + x)], cellIdx[2 * (n * c + x) + 1]) + */ + private int[] cellIdx; + + private final CellData data; + + // The first free idx in data (for writing purposes). + private int idx; + + // THe (complex) cells path. This is indexed exactly like the cells in data (so through cellIdx). + private CellPath[] complexPaths; + + public ComplexCellBlock(Columns columns, int rows, boolean isCounter) + { + this.columns = columns; + + int columnCount = columns.complexColumnCount(); + this.cellIdx = new int[columnCount * 2 * rows]; + + // We start with an estimated 4 cells per complex column. The arrays + // will grow if needed so this is just a somewhat random estimation. + int cellCount = columnCount * 4; + this.data = new CellData(cellCount, isCounter); + this.complexPaths = new CellPath[cellCount]; + } + + public void addCell(int columnIdx, ByteBuffer value, LivenessInfo info, CellPath path, boolean isFirstCell) + { + if (isFirstCell) + cellIdx[columnIdx] = idx; + cellIdx[columnIdx + 1] = idx + 1; + + data.setCell(idx, value, info); + ensureComplexPathsCapacity(idx); + complexPaths[idx] = path; + idx++; + } + + public long unsharedHeapSizeExcludingData() + { + long size = ObjectSizes.sizeOfArray(cellIdx) + + data.unsharedHeapSizeExcludingData() + + ObjectSizes.sizeOfArray(complexPaths); + + for (int i = 0; i < complexPaths.length; i++) + if (complexPaths[i] != null) + size += ((MemtableRowData.BufferCellPath)complexPaths[i]).unsharedHeapSizeExcludingData(); + return size; + } + + public int dataSize() + { + int size = data.dataSize() + cellIdx.length * 4; + + for (int i = 0; i < complexPaths.length; i++) + if (complexPaths[i] != null) + size += complexPaths[i].dataSize(); + + return size; + } + + private void ensureCapacity(int rowToSet) + { + int columnCount = columns.complexColumnCount(); + int originalCapacity = cellIdx.length / (2 * columnCount); + if (rowToSet < originalCapacity) + return; + + int newCapacity = RowDataBlock.computeNewCapacity(originalCapacity, rowToSet); + cellIdx = Arrays.copyOf(cellIdx, newCapacity * 2 * columnCount); + } + + private void ensureComplexPathsCapacity(int idxToSet) + { + int originalCapacity = complexPaths.length; + if (idxToSet < originalCapacity) + return; + + int newCapacity = RowDataBlock.computeNewCapacity(originalCapacity, idxToSet); + complexPaths = Arrays.copyOf(complexPaths, newCapacity); + } + + public void clear() + { + data.clear(); + Arrays.fill(cellIdx, 0); + Arrays.fill(complexPaths, null); + idx = 0; + } + } + + /** + * Simple sublcassing of {@code CellData.ReusableCell} to include the cell path. + */ + private static class ReusableCell extends CellData.ReusableCell + { + private ComplexCellBlock cellBlock; + + ReusableCell setTo(ComplexCellBlock cellBlock, ColumnDefinition column, int idx) + { + this.cellBlock = cellBlock; + super.setTo(cellBlock.data, column, idx); + return this; + } + + @Override + public CellPath path() + { + return cellBlock.complexPaths[idx]; + } + } + + /** + * An iterator over the complex cells of a given row. + * This is used both to iterate over all the (complex) cells of the row, or only on the cells + * of a given column within the row. + */ + static class ReusableIterator extends UnmodifiableIterator + { + private ComplexCellBlock cellBlock; + private final ReusableCell cell = new ReusableCell(); + + // The idx in 'cellBlock' of the row we're iterating over + private int rowIdx; + + // columnIdx is the index in 'columns' of the current column we're iterating over. + // 'endColumnIdx' is the value of 'columnIdx' at which we should stop iterating. + private int columnIdx; + private int endColumnIdx; + + // idx is the index in 'cellBlock.data' of the current cell this iterator is on. 'endIdx' + // is the index in 'cellBlock.data' of the first cell that does not belong to the current + // column we're iterating over (the one pointed by columnIdx). + private int idx; + private int endIdx; + + private ReusableIterator() + { + } + + // Sets the iterator for iterating over the cells of 'column' in 'row' + public ReusableIterator setTo(ComplexRowDataBlock dataBlock, int row, ColumnDefinition column) + { + if (dataBlock == null) + { + this.cellBlock = null; + return null; + } + + this.cellBlock = dataBlock.cellBlock(row); + if (cellBlock == null) + return null; + + rowIdx = dataBlock.cellBlockBase(row); + + columnIdx = dataBlock.columns.complexIdx(column, 0); + if (columnIdx < 0) + return null; + + // We only want the cells of 'column', so stop as soon as we've reach the next column + endColumnIdx = columnIdx + 1; + + resetCellIdx(); + + return endIdx <= idx ? null : this; + } + + // Sets the iterator for iterating over all the cells of 'row' + public ReusableIterator setTo(ComplexRowDataBlock dataBlock, int row) + { + if (dataBlock == null) + { + this.cellBlock = null; + return null; + } + + this.cellBlock = dataBlock.cellBlock(row); + if (cellBlock == null) + return null; + + rowIdx = dataBlock.cellBlockBase(row); + + // We want to iterator over all columns + columnIdx = 0; + endColumnIdx = dataBlock.columns.complexColumnCount(); + + // Not every column might have cells, so set thing up so we're on the + // column having cells (with idx and endIdx sets properly for that column) + findNextColumnWithCells(); + return columnIdx < endColumnIdx ? null : this; + } + + private void findNextColumnWithCells() + { + while (columnIdx < endColumnIdx) + { + resetCellIdx(); + if (idx < endIdx) + return; + ++columnIdx; + } + } + + // Provided that columnIdx and rowIdx are properly set, sets idx to the first + // cells of the pointed column, and endIdx to the first cell not for said column + private void resetCellIdx() + { + int i = rowIdx + 2 * columnIdx; + if (i >= cellBlock.cellIdx.length) + { + idx = 0; + endIdx = 0; + } + else + { + idx = cellBlock.cellIdx[i]; + endIdx = cellBlock.cellIdx[i + 1]; + } + } + + public boolean hasNext() + { + if (cellBlock == null) + return false; + + if (columnIdx >= endColumnIdx) + return false; + + // checks if we have more cells for the current column + if (idx < endIdx) + return true; + + // otherwise, find the next column that has cells. + ++columnIdx; + findNextColumnWithCells(); + + return columnIdx < endColumnIdx; + } + + public Cell next() + { + return cell.setTo(cellBlock, cellBlock.columns.getComplex(columnIdx), idx++); + } + } + + public class CellWriter + { + private final boolean inOrderCells; + + private int base; + private int row; + private int lastColumnIdx; + + public CellWriter(boolean inOrderCells) + { + this.inOrderCells = inOrderCells; + } + + public void addCell(ColumnDefinition column, ByteBuffer value, LivenessInfo info, CellPath path) + { + assert path != null; + + ComplexCellBlock cellBlock = cellBlockForWritting(row); + + lastColumnIdx = columns.complexIdx(column, inOrderCells ? lastColumnIdx : 0); + assert lastColumnIdx >= 0 : "Cannot find column " + column.name + " in " + columns; + + int idx = cellBlockBase(row) + 2 * lastColumnIdx; + + int start = cellBlock.cellIdx[idx]; + int end = cellBlock.cellIdx[idx + 1]; + + cellBlock.addCell(idx, value, info, path, end <= start); + } + + public void setComplexDeletion(ColumnDefinition column, DeletionTime deletionTime) + { + int columnIdx = base + columns.complexIdx(column, 0); + ensureDelTimesCapacity(row); + complexDelTimes.set(columnIdx, deletionTime); + } + + public void endOfRow() + { + base += columns.complexColumnCount(); + lastColumnIdx = 0; + ++row; + } + + public void reset() + { + base = 0; + row = 0; + lastColumnIdx = 0; + clearCellData(); + complexDelTimes.clear(); + } + } +} diff --git a/src/java/org/apache/cassandra/db/composites/CBuilder.java b/src/java/org/apache/cassandra/db/rows/CounterCells.java similarity index 66% rename from src/java/org/apache/cassandra/db/composites/CBuilder.java rename to src/java/org/apache/cassandra/db/rows/CounterCells.java index 39035cbbbe..732f195ae9 100644 --- a/src/java/org/apache/cassandra/db/composites/CBuilder.java +++ b/src/java/org/apache/cassandra/db/rows/CounterCells.java @@ -15,22 +15,18 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -package org.apache.cassandra.db.composites; +package org.apache.cassandra.db.rows; -import java.nio.ByteBuffer; -import java.util.List; +import org.apache.cassandra.db.context.CounterContext; -/** - * A builder of Composite. - */ -public interface CBuilder +public abstract class CounterCells { - public int remainingCount(); + private CounterCells() {} - public CBuilder add(ByteBuffer value); - public CBuilder add(Object value); + private static final CounterContext contextManager = CounterContext.instance(); - public Composite build(); - public Composite buildWith(ByteBuffer value); - public Composite buildWith(List values); + public static boolean hasLegacyShards(Cell cell) + { + return contextManager.hasLegacyShards(cell.value()); + } } diff --git a/src/java/org/apache/cassandra/db/rows/FilteringRow.java b/src/java/org/apache/cassandra/db/rows/FilteringRow.java new file mode 100644 index 0000000000..fb8f448aaa --- /dev/null +++ b/src/java/org/apache/cassandra/db/rows/FilteringRow.java @@ -0,0 +1,121 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.cassandra.db.rows; + +import java.util.Iterator; + +import org.apache.cassandra.config.ColumnDefinition; +import org.apache.cassandra.db.*; +import org.apache.cassandra.db.filter.ColumnFilter; + +public abstract class FilteringRow extends WrappingRow +{ + public static FilteringRow columnsFilteringRow(final Columns toInclude) + { + return new FilteringRow() + { + @Override + protected boolean include(ColumnDefinition column) + { + return toInclude.contains(column); + } + }; + } + + public static FilteringRow columnsFilteringRow(final ColumnFilter toInclude) + { + return new FilteringRow() + { + @Override + protected boolean include(ColumnDefinition column) + { + return toInclude.includes(column); + } + + @Override + protected boolean include(Cell cell) + { + return toInclude.includes(cell); + } + }; + } + + public FilteringRow setTo(Row row) + { + super.setTo(row); + return this; + } + + /** + * The following functions are meant to be overriden based on needs. + */ + protected boolean include(Cell cell) { return true; } + protected boolean include(LivenessInfo info) { return true; } + protected boolean include(DeletionTime dt) { return true; } + protected boolean include(ColumnDefinition column) { return true; } + protected boolean include(ColumnDefinition c, DeletionTime dt) { return true; } + + // Sublcasses that override this should be careful to call the overriden version first, or this might break FilteringRow (i.e. it might not + // filter what it should). + @Override + protected Cell filterCell(Cell cell) + { + return include(cell.column()) && include(cell.livenessInfo()) && include(cell) ? cell : null; + } + + protected DeletionTime filterDeletionTime(DeletionTime deletion) + { + return deletion == null || !include(deletion) + ? DeletionTime.LIVE + : deletion; + } + + @Override + public LivenessInfo primaryKeyLivenessInfo() + { + LivenessInfo info = super.primaryKeyLivenessInfo(); + return include(info) ? info : LivenessInfo.NONE; + } + + @Override + public DeletionTime deletion() + { + DeletionTime deletion = super.deletion(); + return include(deletion) ? deletion : DeletionTime.LIVE; + } + + @Override + public Iterator getCells(ColumnDefinition c) + { + // slightly speed things up if we know we don't care at all about the column + if (!include(c)) + return null; + + return super.getCells(c); + } + + @Override + public DeletionTime getDeletion(ColumnDefinition c) + { + if (!include(c)) + return DeletionTime.LIVE; + + DeletionTime dt = super.getDeletion(c); + return include(c, dt) ? dt : DeletionTime.LIVE; + } +} diff --git a/src/java/org/apache/cassandra/db/rows/FilteringRowIterator.java b/src/java/org/apache/cassandra/db/rows/FilteringRowIterator.java new file mode 100644 index 0000000000..fd1c0a1984 --- /dev/null +++ b/src/java/org/apache/cassandra/db/rows/FilteringRowIterator.java @@ -0,0 +1,126 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.cassandra.db.rows; + +import org.apache.cassandra.db.*; + +public class FilteringRowIterator extends WrappingUnfilteredRowIterator +{ + private final FilteringRow filter; + private Unfiltered next; + + public FilteringRowIterator(UnfilteredRowIterator toFilter) + { + super(toFilter); + this.filter = makeRowFilter(); + } + + // Subclasses that want to filter withing row should overwrite this. Note that since FilteringRow + // is a reusable object, this method won't be called for every filtered row and the same filter will + // be used for every regular rows. However, this still can be called twice if we have a static row + // to filter, because we don't want to use the same object for them as this makes for weird behavior + // if calls to staticRow() are interleaved with hasNext(). + protected FilteringRow makeRowFilter() + { + return null; + } + + protected boolean includeRangeTombstoneMarker(RangeTombstoneMarker marker) + { + return true; + } + + // Allows to modify the range tombstone returned. This is called *after* includeRangeTombstoneMarker has been called. + protected RangeTombstoneMarker filterRangeTombstoneMarker(RangeTombstoneMarker marker, boolean reversed) + { + return marker; + } + + protected boolean includeRow(Row row) + { + return true; + } + + protected boolean includePartitionDeletion(DeletionTime dt) + { + return true; + } + + @Override + public DeletionTime partitionLevelDeletion() + { + DeletionTime dt = wrapped.partitionLevelDeletion(); + return includePartitionDeletion(dt) ? dt : DeletionTime.LIVE; + } + + @Override + public Row staticRow() + { + Row row = super.staticRow(); + if (row == Rows.EMPTY_STATIC_ROW) + return row; + + FilteringRow filter = makeRowFilter(); + if (filter != null) + row = filter.setTo(row); + + return !row.isEmpty() && includeRow(row) ? row : Rows.EMPTY_STATIC_ROW; + } + + @Override + public boolean hasNext() + { + if (next != null) + return true; + + while (super.hasNext()) + { + Unfiltered unfiltered = super.next(); + if (unfiltered.kind() == Unfiltered.Kind.ROW) + { + Row row = filter == null ? (Row) unfiltered : filter.setTo((Row) unfiltered); + if (!row.isEmpty() && includeRow(row)) + { + next = row; + return true; + } + } + else + { + RangeTombstoneMarker marker = (RangeTombstoneMarker) unfiltered; + if (includeRangeTombstoneMarker(marker)) + { + next = filterRangeTombstoneMarker(marker, isReverseOrder()); + return true; + } + } + } + return false; + } + + @Override + public Unfiltered next() + { + if (next == null) + hasNext(); + + Unfiltered toReturn = next; + next = null; + return toReturn; + } +} diff --git a/src/java/org/apache/cassandra/db/rows/LazilyInitializedUnfilteredRowIterator.java b/src/java/org/apache/cassandra/db/rows/LazilyInitializedUnfilteredRowIterator.java new file mode 100644 index 0000000000..6241a8956e --- /dev/null +++ b/src/java/org/apache/cassandra/db/rows/LazilyInitializedUnfilteredRowIterator.java @@ -0,0 +1,103 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.cassandra.db.rows; + +import com.google.common.collect.AbstractIterator; + +import org.apache.cassandra.config.CFMetaData; +import org.apache.cassandra.db.*; + +/** + * Abstract class to create UnfilteredRowIterator that lazily initialize themselves. + * + * This is used during partition range queries when we know the partition key but want + * to defer the initialization of the rest of the UnfilteredRowIterator until we need those informations. + * See {@link BigTableScanner#KeyScanningIterator} for instance. + */ +public abstract class LazilyInitializedUnfilteredRowIterator extends AbstractIterator implements UnfilteredRowIterator +{ + private final DecoratedKey partitionKey; + + private UnfilteredRowIterator iterator; + + public LazilyInitializedUnfilteredRowIterator(DecoratedKey partitionKey) + { + this.partitionKey = partitionKey; + } + + protected abstract UnfilteredRowIterator initializeIterator(); + + private void maybeInit() + { + if (iterator == null) + iterator = initializeIterator(); + } + + public CFMetaData metadata() + { + maybeInit(); + return iterator.metadata(); + } + + public PartitionColumns columns() + { + maybeInit(); + return iterator.columns(); + } + + public boolean isReverseOrder() + { + maybeInit(); + return iterator.isReverseOrder(); + } + + public DecoratedKey partitionKey() + { + return partitionKey; + } + + public DeletionTime partitionLevelDeletion() + { + maybeInit(); + return iterator.partitionLevelDeletion(); + } + + public Row staticRow() + { + maybeInit(); + return iterator.staticRow(); + } + + public RowStats stats() + { + maybeInit(); + return iterator.stats(); + } + + protected Unfiltered computeNext() + { + maybeInit(); + return iterator.hasNext() ? iterator.next() : endOfData(); + } + + public void close() + { + if (iterator != null) + iterator.close(); + } +} diff --git a/src/java/org/apache/cassandra/db/rows/MemtableRowData.java b/src/java/org/apache/cassandra/db/rows/MemtableRowData.java new file mode 100644 index 0000000000..cad0765ada --- /dev/null +++ b/src/java/org/apache/cassandra/db/rows/MemtableRowData.java @@ -0,0 +1,204 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.cassandra.db.rows; + +import java.nio.ByteBuffer; + +import org.apache.cassandra.db.*; +import org.apache.cassandra.utils.ObjectSizes; +import org.apache.cassandra.utils.memory.AbstractAllocator; + +/** + * Row data stored inside a memtable. + * + * This has methods like dataSize and unsharedHeapSizeExcludingData that are + * specific to memtables. + */ +public interface MemtableRowData extends Clusterable +{ + public Columns columns(); + + public int dataSize(); + + // returns the size of the Row and all references on the heap, excluding any costs associated with byte arrays + // that would be allocated by a clone operation, as these will be accounted for by the allocator + public long unsharedHeapSizeExcludingData(); + + public interface ReusableRow extends Row + { + public ReusableRow setTo(MemtableRowData rowData); + } + + public class BufferRowData implements MemtableRowData + { + private static final long EMPTY_SIZE = ObjectSizes.measure(new BufferRowData(null, LivenessInfo.NONE, DeletionTime.LIVE, null)); + + private final Clustering clustering; + private final LivenessInfo livenessInfo; + private final DeletionTime deletion; + private final RowDataBlock dataBlock; + + public BufferRowData(Clustering clustering, LivenessInfo livenessInfo, DeletionTime deletion, RowDataBlock dataBlock) + { + this.clustering = clustering; + this.livenessInfo = livenessInfo.takeAlias(); + this.deletion = deletion.takeAlias(); + this.dataBlock = dataBlock; + } + + public Clustering clustering() + { + return clustering; + } + + public Columns columns() + { + return dataBlock.columns(); + } + + public int dataSize() + { + return clustering.dataSize() + livenessInfo.dataSize() + deletion.dataSize() + dataBlock.dataSize(); + } + + public long unsharedHeapSizeExcludingData() + { + return EMPTY_SIZE + + (clustering == Clustering.STATIC_CLUSTERING ? 0 : ((BufferClustering)clustering).unsharedHeapSizeExcludingData()) + + dataBlock.unsharedHeapSizeExcludingData(); + } + + public static ReusableRow createReusableRow() + { + return new BufferRow(); + } + + private static class BufferRow extends AbstractReusableRow implements ReusableRow + { + private BufferRowData rowData; + + private BufferRow() + { + } + + public ReusableRow setTo(MemtableRowData rowData) + { + assert rowData instanceof BufferRowData; + this.rowData = (BufferRowData)rowData; + return this; + } + + protected RowDataBlock data() + { + return rowData.dataBlock; + } + + protected int row() + { + return 0; + } + + public Clustering clustering() + { + return rowData.clustering; + } + + public LivenessInfo primaryKeyLivenessInfo() + { + return rowData.livenessInfo; + } + + public DeletionTime deletion() + { + return rowData.deletion; + } + } + } + + public class BufferClustering extends Clustering + { + private static final long EMPTY_SIZE = ObjectSizes.measure(new BufferClustering(0)); + + private final ByteBuffer[] values; + + public BufferClustering(int size) + { + this.values = new ByteBuffer[size]; + } + + public void setClusteringValue(int i, ByteBuffer value) + { + values[i] = value; + } + + public int size() + { + return values.length; + } + + public ByteBuffer get(int i) + { + return values[i]; + } + + public ByteBuffer[] getRawValues() + { + return values; + } + + public long unsharedHeapSizeExcludingData() + { + return EMPTY_SIZE + ObjectSizes.sizeOnHeapExcludingData(values); + } + + @Override + public long unsharedHeapSize() + { + return EMPTY_SIZE + ObjectSizes.sizeOnHeapOf(values); + } + + public Clustering takeAlias() + { + return this; + } + } + + public class BufferCellPath extends CellPath.SimpleCellPath + { + private static final long EMPTY_SIZE = ObjectSizes.measure(new BufferCellPath(new ByteBuffer[0])); + + private BufferCellPath(ByteBuffer[] values) + { + super(values); + } + + public static BufferCellPath clone(CellPath path, AbstractAllocator allocator) + { + int size = path.size(); + ByteBuffer[] values = new ByteBuffer[size]; + for (int i = 0; i < size; i++) + values[i] = allocator.clone(path.get(0)); + return new BufferCellPath(values); + } + + public long unsharedHeapSizeExcludingData() + { + return EMPTY_SIZE + ObjectSizes.sizeOnHeapExcludingData(values); + } + } +} diff --git a/src/java/org/apache/cassandra/db/rows/RangeTombstoneBoundMarker.java b/src/java/org/apache/cassandra/db/rows/RangeTombstoneBoundMarker.java new file mode 100644 index 0000000000..b5ac19bec1 --- /dev/null +++ b/src/java/org/apache/cassandra/db/rows/RangeTombstoneBoundMarker.java @@ -0,0 +1,156 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.cassandra.db.rows; + +import java.nio.ByteBuffer; +import java.security.MessageDigest; +import java.util.Objects; + +import org.apache.cassandra.config.CFMetaData; +import org.apache.cassandra.db.*; + +/** + * A range tombstone marker that indicates the bound of a range tombstone (start or end). + */ +public class RangeTombstoneBoundMarker extends AbstractRangeTombstoneMarker +{ + private final DeletionTime deletion; + + public RangeTombstoneBoundMarker(RangeTombstone.Bound bound, DeletionTime deletion) + { + super(bound); + assert bound.kind().isBound(); + this.deletion = deletion; + } + + public RangeTombstoneBoundMarker(Slice.Bound bound, DeletionTime deletion) + { + this(new RangeTombstone.Bound(bound.kind(), bound.getRawValues()), deletion); + } + + public static RangeTombstoneBoundMarker inclusiveStart(ClusteringPrefix clustering, DeletionTime deletion) + { + return new RangeTombstoneBoundMarker(new RangeTombstone.Bound(RangeTombstone.Bound.Kind.INCL_START_BOUND, clustering.getRawValues()), deletion); + } + + public static RangeTombstoneBoundMarker inclusiveEnd(ClusteringPrefix clustering, DeletionTime deletion) + { + return new RangeTombstoneBoundMarker(new RangeTombstone.Bound(RangeTombstone.Bound.Kind.INCL_END_BOUND, clustering.getRawValues()), deletion); + } + + public static RangeTombstoneBoundMarker inclusiveOpen(boolean reversed, ByteBuffer[] boundValues, DeletionTime deletion) + { + RangeTombstone.Bound bound = RangeTombstone.Bound.inclusiveOpen(reversed, boundValues); + return new RangeTombstoneBoundMarker(bound, deletion); + } + + public static RangeTombstoneBoundMarker exclusiveOpen(boolean reversed, ByteBuffer[] boundValues, DeletionTime deletion) + { + RangeTombstone.Bound bound = RangeTombstone.Bound.exclusiveOpen(reversed, boundValues); + return new RangeTombstoneBoundMarker(bound, deletion); + } + + public static RangeTombstoneBoundMarker inclusiveClose(boolean reversed, ByteBuffer[] boundValues, DeletionTime deletion) + { + RangeTombstone.Bound bound = RangeTombstone.Bound.inclusiveClose(reversed, boundValues); + return new RangeTombstoneBoundMarker(bound, deletion); + } + + public static RangeTombstoneBoundMarker exclusiveClose(boolean reversed, ByteBuffer[] boundValues, DeletionTime deletion) + { + RangeTombstone.Bound bound = RangeTombstone.Bound.exclusiveClose(reversed, boundValues); + return new RangeTombstoneBoundMarker(bound, deletion); + } + + public boolean isBoundary() + { + return false; + } + + /** + * The deletion time for the range tombstone this is a bound of. + */ + public DeletionTime deletionTime() + { + return deletion; + } + + public boolean isOpen(boolean reversed) + { + return bound.kind().isOpen(reversed); + } + + public boolean isClose(boolean reversed) + { + return bound.kind().isClose(reversed); + } + + public DeletionTime openDeletionTime(boolean reversed) + { + if (!isOpen(reversed)) + throw new IllegalStateException(); + return deletion; + } + + public DeletionTime closeDeletionTime(boolean reversed) + { + if (isOpen(reversed)) + throw new IllegalStateException(); + return deletion; + } + + public void copyTo(RangeTombstoneMarker.Writer writer) + { + copyBoundTo(writer); + writer.writeBoundDeletion(deletion); + writer.endOfMarker(); + } + + public void digest(MessageDigest digest) + { + bound.digest(digest); + deletion.digest(digest); + } + + public String toString(CFMetaData metadata) + { + StringBuilder sb = new StringBuilder(); + sb.append("Marker "); + sb.append(bound.toString(metadata)); + sb.append("@").append(deletion.markedForDeleteAt()); + return sb.toString(); + } + + @Override + public boolean equals(Object other) + { + if(!(other instanceof RangeTombstoneBoundMarker)) + return false; + + RangeTombstoneBoundMarker that = (RangeTombstoneBoundMarker)other; + return this.bound.equals(that.bound) + && this.deletion.equals(that.deletion); + } + + @Override + public int hashCode() + { + return Objects.hash(bound, deletion); + } +} + diff --git a/src/java/org/apache/cassandra/db/rows/RangeTombstoneBoundaryMarker.java b/src/java/org/apache/cassandra/db/rows/RangeTombstoneBoundaryMarker.java new file mode 100644 index 0000000000..1140d40a1b --- /dev/null +++ b/src/java/org/apache/cassandra/db/rows/RangeTombstoneBoundaryMarker.java @@ -0,0 +1,173 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.cassandra.db.rows; + +import java.nio.ByteBuffer; +import java.security.MessageDigest; +import java.util.Objects; + +import org.apache.cassandra.config.CFMetaData; +import org.apache.cassandra.db.*; + +/** + * A range tombstone marker that represents a boundary between 2 range tombstones (i.e. it closes one range and open another). + */ +public class RangeTombstoneBoundaryMarker extends AbstractRangeTombstoneMarker +{ + private final DeletionTime endDeletion; + private final DeletionTime startDeletion; + + public RangeTombstoneBoundaryMarker(RangeTombstone.Bound bound, DeletionTime endDeletion, DeletionTime startDeletion) + { + super(bound); + assert bound.kind().isBoundary(); + this.endDeletion = endDeletion; + this.startDeletion = startDeletion; + } + + public static RangeTombstoneBoundaryMarker exclusiveCloseInclusiveOpen(boolean reversed, ByteBuffer[] boundValues, DeletionTime closeDeletion, DeletionTime openDeletion) + { + RangeTombstone.Bound bound = RangeTombstone.Bound.exclusiveCloseInclusiveOpen(reversed, boundValues); + DeletionTime endDeletion = reversed ? openDeletion : closeDeletion; + DeletionTime startDeletion = reversed ? closeDeletion : openDeletion; + return new RangeTombstoneBoundaryMarker(bound, endDeletion, startDeletion); + } + + public static RangeTombstoneBoundaryMarker inclusiveCloseExclusiveOpen(boolean reversed, ByteBuffer[] boundValues, DeletionTime closeDeletion, DeletionTime openDeletion) + { + RangeTombstone.Bound bound = RangeTombstone.Bound.inclusiveCloseExclusiveOpen(reversed, boundValues); + DeletionTime endDeletion = reversed ? openDeletion : closeDeletion; + DeletionTime startDeletion = reversed ? closeDeletion : openDeletion; + return new RangeTombstoneBoundaryMarker(bound, endDeletion, startDeletion); + } + + public boolean isBoundary() + { + return true; + } + + /** + * The deletion time for the range tombstone this boundary ends (in clustering order). + */ + public DeletionTime endDeletionTime() + { + return endDeletion; + } + + /** + * The deletion time for the range tombstone this boundary starts (in clustering order). + */ + public DeletionTime startDeletionTime() + { + return startDeletion; + } + + public DeletionTime closeDeletionTime(boolean reversed) + { + return reversed ? startDeletion : endDeletion; + } + + public DeletionTime openDeletionTime(boolean reversed) + { + return reversed ? endDeletion : startDeletion; + } + + public boolean isOpen(boolean reversed) + { + // A boundary always open one side + return true; + } + + public boolean isClose(boolean reversed) + { + // A boundary always close one side + return true; + } + + public static boolean isBoundary(ClusteringComparator comparator, Slice.Bound close, Slice.Bound open) + { + if (!comparator.isOnSameClustering(close, open)) + return false; + + // If both bound are exclusive, then it's not a boundary, otherwise it is one. + // Note that most code should never call this with 2 inclusive bound: this would mean we had + // 2 RTs that were overlapping and RangeTombstoneList don't create that. However, old + // code was generating that so supporting this case helps dealing with backward compatibility. + return close.isInclusive() || open.isInclusive(); + } + + // Please note that isBoundary *must* have been called (and returned true) before this is called. + public static RangeTombstoneBoundaryMarker makeBoundary(boolean reversed, Slice.Bound close, Slice.Bound open, DeletionTime closeDeletion, DeletionTime openDeletion) + { + boolean isExclusiveClose = close.isExclusive() || (close.isInclusive() && open.isInclusive() && openDeletion.supersedes(closeDeletion)); + return isExclusiveClose + ? exclusiveCloseInclusiveOpen(reversed, close.getRawValues(), closeDeletion, openDeletion) + : inclusiveCloseExclusiveOpen(reversed, close.getRawValues(), closeDeletion, openDeletion); + } + + public RangeTombstoneBoundMarker createCorrespondingCloseBound(boolean reversed) + { + return new RangeTombstoneBoundMarker(bound.withNewKind(bound.kind().closeBoundOfBoundary(reversed)), endDeletion); + } + + public RangeTombstoneBoundMarker createCorrespondingOpenBound(boolean reversed) + { + return new RangeTombstoneBoundMarker(bound.withNewKind(bound.kind().openBoundOfBoundary(reversed)), startDeletion); + } + + public void copyTo(RangeTombstoneMarker.Writer writer) + { + copyBoundTo(writer); + writer.writeBoundaryDeletion(endDeletion, startDeletion); + writer.endOfMarker(); + } + + public void digest(MessageDigest digest) + { + bound.digest(digest); + endDeletion.digest(digest); + startDeletion.digest(digest); + } + + public String toString(CFMetaData metadata) + { + StringBuilder sb = new StringBuilder(); + sb.append("Marker "); + sb.append(bound.toString(metadata)); + sb.append("@").append(endDeletion.markedForDeleteAt()).append("-").append(startDeletion.markedForDeleteAt()); + return sb.toString(); + } + + @Override + public boolean equals(Object other) + { + if(!(other instanceof RangeTombstoneBoundaryMarker)) + return false; + + RangeTombstoneBoundaryMarker that = (RangeTombstoneBoundaryMarker)other; + return this.bound.equals(that.bound) + && this.endDeletion.equals(that.endDeletion) + && this.startDeletion.equals(that.startDeletion); + } + + @Override + public int hashCode() + { + return Objects.hash(bound, endDeletion, startDeletion); + } +} diff --git a/src/java/org/apache/cassandra/db/rows/RangeTombstoneMarker.java b/src/java/org/apache/cassandra/db/rows/RangeTombstoneMarker.java new file mode 100644 index 0000000000..1a506d5ade --- /dev/null +++ b/src/java/org/apache/cassandra/db/rows/RangeTombstoneMarker.java @@ -0,0 +1,283 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.cassandra.db.rows; + +import java.nio.ByteBuffer; +import java.util.*; + +import org.apache.cassandra.config.CFMetaData; +import org.apache.cassandra.db.*; + +/** + * A marker for a range tombstone bound. + *

+ * There is 2 types of markers: bounds (see {@link RangeTombstoneBound}) and boundaries (see {@link RangeTombstoneBoundary}). + */ +public interface RangeTombstoneMarker extends Unfiltered +{ + @Override + public RangeTombstone.Bound clustering(); + + public boolean isBoundary(); + + public void copyTo(RangeTombstoneMarker.Writer writer); + + public boolean isOpen(boolean reversed); + public boolean isClose(boolean reversed); + public DeletionTime openDeletionTime(boolean reversed); + public DeletionTime closeDeletionTime(boolean reversed); + + public interface Writer extends Slice.Bound.Writer + { + public void writeBoundDeletion(DeletionTime deletion); + public void writeBoundaryDeletion(DeletionTime endDeletion, DeletionTime startDeletion); + public void endOfMarker(); + } + + public static class Builder implements Writer + { + private final ByteBuffer[] values; + private int size; + + private RangeTombstone.Bound.Kind kind; + private DeletionTime firstDeletion; + private DeletionTime secondDeletion; + + public Builder(int maxClusteringSize) + { + this.values = new ByteBuffer[maxClusteringSize]; + } + + public void writeClusteringValue(ByteBuffer value) + { + values[size++] = value; + } + + public void writeBoundKind(RangeTombstone.Bound.Kind kind) + { + this.kind = kind; + } + + public void writeBoundDeletion(DeletionTime deletion) + { + firstDeletion = deletion; + } + + public void writeBoundaryDeletion(DeletionTime endDeletion, DeletionTime startDeletion) + { + firstDeletion = endDeletion; + secondDeletion = startDeletion; + } + + public void endOfMarker() + { + } + + public RangeTombstoneMarker build() + { + assert kind != null : "Nothing has been written"; + if (kind.isBoundary()) + return new RangeTombstoneBoundaryMarker(new RangeTombstone.Bound(kind, Arrays.copyOfRange(values, 0, size)), firstDeletion, secondDeletion); + else + return new RangeTombstoneBoundMarker(new RangeTombstone.Bound(kind, Arrays.copyOfRange(values, 0, size)), firstDeletion); + } + + public Builder reset() + { + Arrays.fill(values, null); + size = 0; + kind = null; + return this; + } + } + + /** + * Utility class to help merging range tombstone markers coming from multiple inputs (UnfilteredRowIterators). + *

+ * The assumption that each individual input must validate and that we must preserve in the output is that every + * open marker has a corresponding close marker with the exact same deletion info, and that there is no other range + * tombstone marker between those open and close marker (of course, they could be rows in between). In other word, + * for any {@code UnfilteredRowIterator}, you only ever have to remenber the last open marker (if any) to have the + * full picture of what is deleted by range tombstones at any given point of iterating that iterator. + *

+ * Note that this class can merge both forward and reverse iterators. To deal with reverse, we just reverse how we + * deal with open and close markers (in forward order, we'll get open-close, open-close, ..., while in reverse we'll + * get close-open, close-open, ...). + */ + public static class Merger + { + // Boundaries sorts like the bound that have their equivalent "inclusive" part and that's the main action we + // care about as far as merging goes. So MergedKind just group those as the same case, and tell us whether + // we're dealing with an open or a close (based on whether we're dealing with reversed iterators or not). + // Really this enum is just a convenience for merging. + private enum MergedKind + { + INCL_OPEN, EXCL_CLOSE, EXCL_OPEN, INCL_CLOSE; + + public static MergedKind forBound(RangeTombstone.Bound bound, boolean reversed) + { + switch (bound.kind()) + { + case INCL_START_BOUND: + case EXCL_END_INCL_START_BOUNDARY: + return reversed ? INCL_CLOSE : INCL_OPEN; + case EXCL_END_BOUND: + return reversed ? EXCL_OPEN : EXCL_CLOSE; + case EXCL_START_BOUND: + return reversed ? EXCL_CLOSE : EXCL_OPEN; + case INCL_END_EXCL_START_BOUNDARY: + case INCL_END_BOUND: + return reversed ? INCL_OPEN : INCL_CLOSE; + } + throw new AssertionError(); + } + } + + private final CFMetaData metadata; + private final UnfilteredRowIterators.MergeListener listener; + private final DeletionTime partitionDeletion; + private final boolean reversed; + + private RangeTombstone.Bound bound; + private final RangeTombstoneMarker[] markers; + + // For each iterator, what is the currently open marker deletion time (or null if there is no open marker on that iterator) + private final DeletionTime[] openMarkers; + // The index in openMarkers of the "biggest" marker, the one with the biggest deletion time. Is < 0 iff there is no open + // marker on any iterator. + private int biggestOpenMarker = -1; + + public Merger(CFMetaData metadata, int size, DeletionTime partitionDeletion, boolean reversed, UnfilteredRowIterators.MergeListener listener) + { + this.metadata = metadata; + this.listener = listener; + this.partitionDeletion = partitionDeletion; + this.reversed = reversed; + + this.markers = new RangeTombstoneMarker[size]; + this.openMarkers = new DeletionTime[size]; + } + + public void clear() + { + Arrays.fill(markers, null); + } + + public void add(int i, RangeTombstoneMarker marker) + { + bound = marker.clustering(); + markers[i] = marker; + } + + public RangeTombstoneMarker merge() + { + /* + * Merging of range tombstones works this way: + * 1) We remember what is the currently open marker in the merged stream + * 2) We update our internal states of what range is opened on the input streams based on the new markers to merge + * 3) We compute what should be the state in the merge stream after 2) + * 4) We return what marker should be issued on the merged stream based on the difference between the state from 1) and 3) + */ + + DeletionTime previousDeletionTimeInMerged = currentOpenDeletionTimeInMerged(); + + updateOpenMarkers(); + + DeletionTime newDeletionTimeInMerged = currentOpenDeletionTimeInMerged(); + if (previousDeletionTimeInMerged.equals(newDeletionTimeInMerged)) + return null; + + ByteBuffer[] values = bound.getRawValues(); + + RangeTombstoneMarker merged; + switch (MergedKind.forBound(bound, reversed)) + { + case INCL_OPEN: + merged = previousDeletionTimeInMerged.isLive() + ? RangeTombstoneBoundMarker.inclusiveOpen(reversed, values, newDeletionTimeInMerged) + : RangeTombstoneBoundaryMarker.exclusiveCloseInclusiveOpen(reversed, values, previousDeletionTimeInMerged, newDeletionTimeInMerged); + break; + case EXCL_CLOSE: + merged = newDeletionTimeInMerged.isLive() + ? RangeTombstoneBoundMarker.exclusiveClose(reversed, values, previousDeletionTimeInMerged) + : RangeTombstoneBoundaryMarker.exclusiveCloseInclusiveOpen(reversed, values, previousDeletionTimeInMerged, newDeletionTimeInMerged); + break; + case EXCL_OPEN: + merged = previousDeletionTimeInMerged.isLive() + ? RangeTombstoneBoundMarker.exclusiveOpen(reversed, values, newDeletionTimeInMerged) + : RangeTombstoneBoundaryMarker.inclusiveCloseExclusiveOpen(reversed, values, previousDeletionTimeInMerged, newDeletionTimeInMerged); + break; + case INCL_CLOSE: + merged = newDeletionTimeInMerged.isLive() + ? RangeTombstoneBoundMarker.inclusiveClose(reversed, values, previousDeletionTimeInMerged) + : RangeTombstoneBoundaryMarker.inclusiveCloseExclusiveOpen(reversed, values, previousDeletionTimeInMerged, newDeletionTimeInMerged); + break; + default: + throw new AssertionError(); + } + + if (listener != null) + listener.onMergedRangeTombstoneMarkers(merged, markers); + + return merged; + } + + private DeletionTime currentOpenDeletionTimeInMerged() + { + if (biggestOpenMarker < 0) + return DeletionTime.LIVE; + + DeletionTime biggestDeletionTime = openMarkers[biggestOpenMarker]; + // it's only open in the merged iterator if it's not shadowed by the partition level deletion + return partitionDeletion.supersedes(biggestDeletionTime) ? DeletionTime.LIVE : biggestDeletionTime.takeAlias(); + } + + private void updateOpenMarkers() + { + for (int i = 0; i < markers.length; i++) + { + RangeTombstoneMarker marker = markers[i]; + if (marker == null) + continue; + + // Note that we can have boundaries that are both open and close, but in that case all we care about + // is what it the open deletion after the marker, so we favor the opening part in this case. + if (marker.isOpen(reversed)) + openMarkers[i] = marker.openDeletionTime(reversed).takeAlias(); + else + openMarkers[i] = null; + } + + // Recompute what is now the biggest open marker + biggestOpenMarker = -1; + for (int i = 0; i < openMarkers.length; i++) + { + if (openMarkers[i] != null && (biggestOpenMarker < 0 || openMarkers[i].supersedes(openMarkers[biggestOpenMarker]))) + biggestOpenMarker = i; + } + } + + public DeletionTime activeDeletion() + { + DeletionTime openMarker = currentOpenDeletionTimeInMerged(); + // We only have an open marker in the merged stream if it's not shadowed by the partition deletion (which can be LIVE itself), so + // if have an open marker, we know it's the "active" deletion for the merged stream. + return openMarker.isLive() ? partitionDeletion : openMarker; + } + } +} diff --git a/src/java/org/apache/cassandra/db/rows/ReusableRow.java b/src/java/org/apache/cassandra/db/rows/ReusableRow.java new file mode 100644 index 0000000000..0135afc3a4 --- /dev/null +++ b/src/java/org/apache/cassandra/db/rows/ReusableRow.java @@ -0,0 +1,104 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.cassandra.db.rows; + +import java.nio.ByteBuffer; + +import org.apache.cassandra.db.*; + +public class ReusableRow extends AbstractReusableRow +{ + private final ReusableClustering clustering; + + private final ReusableLivenessInfo liveness = new ReusableLivenessInfo(); + + private DeletionTime deletion = DeletionTime.LIVE; + + private final RowDataBlock data; + private final Writer writer; + + public ReusableRow(int clusteringSize, Columns columns, boolean inOrderCells, boolean isCounter) + { + this.clustering = new ReusableClustering(clusteringSize); + this.data = new RowDataBlock(columns, 1, false, isCounter); + this.writer = new Writer(data, inOrderCells); + } + + protected RowDataBlock data() + { + return data; + } + + protected int row() + { + return 0; + } + + public Clustering clustering() + { + return clustering; + } + + public LivenessInfo primaryKeyLivenessInfo() + { + return liveness; + } + + public DeletionTime deletion() + { + return deletion; + } + + public Row.Writer writer() + { + return writer.reset(); + } + + private class Writer extends RowDataBlock.Writer + { + public Writer(RowDataBlock data, boolean inOrderCells) + { + super(data, inOrderCells); + } + + public void writeClusteringValue(ByteBuffer buffer) + { + clustering.writer().writeClusteringValue(buffer); + } + + public void writePartitionKeyLivenessInfo(LivenessInfo info) + { + ReusableRow.this.liveness.setTo(info); + } + + public void writeRowDeletion(DeletionTime deletion) + { + ReusableRow.this.deletion = deletion; + } + + @Override + public Writer reset() + { + super.reset(); + clustering.reset(); + liveness.reset(); + deletion = DeletionTime.LIVE; + return this; + } + } +} diff --git a/src/java/org/apache/cassandra/db/rows/Row.java b/src/java/org/apache/cassandra/db/rows/Row.java new file mode 100644 index 0000000000..545da7a8ad --- /dev/null +++ b/src/java/org/apache/cassandra/db/rows/Row.java @@ -0,0 +1,555 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.cassandra.db.rows; + +import java.nio.ByteBuffer; +import java.util.*; + +import com.google.common.collect.Iterators; + +import org.apache.cassandra.config.CFMetaData; +import org.apache.cassandra.config.ColumnDefinition; +import org.apache.cassandra.db.*; +import org.apache.cassandra.utils.MergeIterator; +import org.apache.cassandra.utils.SearchIterator; + +/** + * Storage engine representation of a row. + * + * A row is identified by it's clustering column values (it's an Unfiltered), + * has row level informations (deletion and partition key liveness infos (see below)) + * and contains data (Cells) regarding the columns it contains. + * + * A row implements {@code WithLivenessInfo} and has thus a timestamp, ttl and + * local deletion time. Those information do not apply to the row content, they + * apply to the partition key columns. In other words, the timestamp is the + * timestamp for the partition key columns: it is what allows to distinguish + * between a dead row, and a live row but for which only the partition key columns + * are set. The ttl and local deletion time information are for the case where + * a TTL is set on those partition key columns. Note however that a row can have + * live cells but no partition key columns timestamp, because said timestamp (and + * its corresponding ttl) is only set on INSERT (not UPDATE). + */ +public interface Row extends Unfiltered, Iterable, Aliasable +{ + /** + * The clustering values for this row. + */ + @Override + public Clustering clustering(); + + /** + * The columns this row contains. + * + * Note that this is actually a superset of the columns the row contains. The row + * may not have values for each of those columns, but it can't have values for other + * columns. + * + * @return a superset of the columns contained in this row. + */ + public Columns columns(); + + /** + * The row deletion. + * + * This correspond to the last row deletion done on this row. + * + * @return the row deletion. + */ + public DeletionTime deletion(); + + /** + * Liveness information for the primary key columns of this row. + *

+ * As a row is uniquely identified by its primary key, all its primary key columns + * share the same {@code LivenessInfo}. This liveness information is what allows us + * to distinguish between a dead row (it has no live cells and its primary key liveness + * info has no timestamp) and a live row but where all non PK columns are null (it has no + * live cells, but its primary key liveness has a timestamp). Please note that the ttl + * (and local deletion time) of the PK liveness information only apply to the + * liveness info timestamp, and not to the content of the row. Also note that because + * in practice there is not way to only delete the primary key columns (without deleting + * the row itself), the returned {@code LivenessInfo} can only have a local deletion time + * if it has a TTL. + *

+ * Lastly, note that it is possible for a row to have live cells but no PK liveness + * info timestamp, because said timestamp is only set on {@code INSERT} (which makes sense + * in itself, see #6782) but live cells can be add through {@code UPDATE} even if the row + * wasn't pre-existing (which users are encouraged not to do, but we can't validate). + */ + public LivenessInfo primaryKeyLivenessInfo(); + + /** + * Whether the row correspond to a static row or not. + * + * @return whether the row correspond to a static row or not. + */ + public boolean isStatic(); + + /** + * Whether the row has no information whatsoever. This means no row infos + * (timestamp, ttl, deletion), no cells and no complex deletion info. + * + * @return {@code true} if the row has no data whatsoever, {@code false} otherwise. + */ + public boolean isEmpty(); + + /** + * Whether the row has some live information (i.e. it's not just deletion informations). + */ + public boolean hasLiveData(int nowInSec); + + /** + * Whether or not this row contains any deletion for a complex column. That is if + * there is at least one column for which {@code getDeletion} returns a non + * live deletion time. + */ + public boolean hasComplexDeletion(); + + /** + * Returns a cell for a simple column. + * + * Calls to this method are allowed to return the same Cell object, and hence the returned + * object is only valid until the next getCell/getCells call on the same Row object. You will need + * to copy the returned data if you plan on using a reference to the Cell object + * longer than that. + * + * @param c the simple column for which to fetch the cell. + * @return the corresponding cell or {@code null} if the row has no such cell. + */ + public Cell getCell(ColumnDefinition c); + + /** + * Return a cell for a given complex column and cell path. + * + * Calls to this method are allowed to return the same Cell object, and hence the returned + * object is only valid until the next getCell/getCells call on the same Row object. You will need + * to copy the returned data if you plan on using a reference to the Cell object + * longer than that. + * + * @param c the complex column for which to fetch the cell. + * @param path the cell path for which to fetch the cell. + * @return the corresponding cell or {@code null} if the row has no such cell. + */ + public Cell getCell(ColumnDefinition c, CellPath path); + + /** + * Returns an iterator on the cells of a complex column c. + * + * Calls to this method are allowed to return the same iterator object, and + * hence the returned object is only valid until the next getCell/getCells call + * on the same Row object. You will need to copy the returned data if you + * plan on using a reference to the Cell object longer than that. + * + * @param c the complex column for which to fetch the cells. + * @return an iterator on the cells of complex column {@code c} or {@code null} if the row has no + * cells for that column. + */ + public Iterator getCells(ColumnDefinition c); + + /** + * Deletion informations for complex columns. + * + * @param c the complex column for which to fetch deletion info. + * @return the deletion time for complex column {@code c} in this row. + */ + public DeletionTime getDeletion(ColumnDefinition c); + + /** + * An iterator over the cells of this row. + * + * The iterator guarantees that for 2 rows of the same partition, columns + * are returned in a consistent order in the sense that if the cells for + * column c1 is returned before the cells for column c2 by the first iterator, + * it is also the case for the 2nd iterator. + * + * The object returned by a call to next() is only guaranteed to be valid until + * the next call to hasNext() or next(). If a consumer wants to keep a + * reference on the returned Cell objects for longer than the iteration, it must + * make a copy of it explicitly. + * + * @return an iterator over the cells of this row. + */ + public Iterator iterator(); + + /** + * An iterator to efficiently search data for a given column. + * + * @return a search iterator for the cells of this row. + */ + public SearchIterator searchIterator(); + + /** + * Copy this row to the provided writer. + * + * @param writer the row writer to write this row to. + */ + public void copyTo(Row.Writer writer); + + public String toString(CFMetaData metadata, boolean fullDetails); + + /** + * Interface for writing a row. + *

+ * Clients of this interface should abid to the following assumptions: + * 1) if the row has a non empty clustering (it's not a static one and it doesn't belong to a table without + * clustering columns), then that clustering should be the first thing written (through + * {@link ClusteringPrefix.Writer#writeClusteringValue})). + * 2) for a given complex column, calls to {@link #writeCell} are performed consecutively (without + * any call to {@code writeCell} for another column intermingled) and in {@code CellPath} order. + * 3) {@link #endOfRow} is always called to end the writing of a given row. + */ + public interface Writer extends ClusteringPrefix.Writer + { + /** + * Writes the livness information for the partition key columns of this row. + * + * This call is optional: skipping it is equivalent to calling {@code writePartitionKeyLivenessInfo(LivenessInfo.NONE)}. + * + * @param info the liveness information for the partition key columns of the written row. + */ + public void writePartitionKeyLivenessInfo(LivenessInfo info); + + /** + * Writes the deletion information for this row. + * + * This call is optional and can be skipped if the row is not deleted. + * + * @param deletion the row deletion time, or {@code DeletionTime.LIVE} if the row isn't deleted. + */ + public void writeRowDeletion(DeletionTime deletion); + + /** + * Writes a cell to the writer. + * + * As mentionned above, add cells for a given column should be added consecutively (and in {@code CellPath} order for complex columns). + * + * @param column the column for the written cell. + * @param isCounter whether or not this is a counter cell. + * @param value the value for the cell. For tombstones, which don't have values, this should be an empty buffer. + * @param info the cell liveness information. + * @param path the {@link CellPath} for complex cells and {@code null} for regular cells. + */ + public void writeCell(ColumnDefinition column, boolean isCounter, ByteBuffer value, LivenessInfo info, CellPath path); + + /** + * Writes a deletion for a complex column, that is one that apply to all cells of the complex column. + * + * @param column the (complex) column this is a deletion for. + * @param complexDeletion the deletion time. + */ + public void writeComplexDeletion(ColumnDefinition column, DeletionTime complexDeletion); + + /** + * Should be called to indicates that the row has been fully written. + */ + public void endOfRow(); + } + + /** + * Utility class to help merging rows from multiple inputs (UnfilteredRowIterators). + */ + public abstract static class Merger + { + private final CFMetaData metadata; + private final int nowInSec; + private final UnfilteredRowIterators.MergeListener listener; + private final Columns columns; + + private Clustering clustering; + private final Row[] rows; + private int rowsToMerge; + + private LivenessInfo rowInfo = LivenessInfo.NONE; + private DeletionTime rowDeletion = DeletionTime.LIVE; + + private final Cell[] cells; + private final List> complexCells; + private final ComplexColumnReducer complexReducer = new ComplexColumnReducer(); + + // For the sake of the listener if there is one + private final DeletionTime[] complexDelTimes; + + private boolean signaledListenerForRow; + + public static Merger createStatic(CFMetaData metadata, int size, int nowInSec, Columns columns, UnfilteredRowIterators.MergeListener listener) + { + return new StaticMerger(metadata, size, nowInSec, columns, listener); + } + + public static Merger createRegular(CFMetaData metadata, int size, int nowInSec, Columns columns, UnfilteredRowIterators.MergeListener listener) + { + return new RegularMerger(metadata, size, nowInSec, columns, listener); + } + + protected Merger(CFMetaData metadata, int size, int nowInSec, Columns columns, UnfilteredRowIterators.MergeListener listener) + { + this.metadata = metadata; + this.nowInSec = nowInSec; + this.listener = listener; + this.columns = columns; + this.rows = new Row[size]; + this.complexCells = new ArrayList<>(size); + + this.cells = new Cell[size]; + this.complexDelTimes = listener == null ? null : new DeletionTime[size]; + } + + public void clear() + { + Arrays.fill(rows, null); + Arrays.fill(cells, null); + if (complexDelTimes != null) + Arrays.fill(complexDelTimes, null); + complexCells.clear(); + rowsToMerge = 0; + + rowInfo = LivenessInfo.NONE; + rowDeletion = DeletionTime.LIVE; + + signaledListenerForRow = false; + } + + public void add(int i, Row row) + { + clustering = row.clustering(); + rows[i] = row; + ++rowsToMerge; + } + + protected abstract Row.Writer getWriter(); + protected abstract Row getRow(); + + public Row merge(DeletionTime activeDeletion) + { + // If for this clustering we have only one row version and have no activeDeletion (i.e. nothing to filter out), + // then we can just return that single row (we also should have no listener) + if (rowsToMerge == 1 && activeDeletion.isLive() && listener == null) + { + for (int i = 0; i < rows.length; i++) + if (rows[i] != null) + return rows[i]; + throw new AssertionError(); + } + + Row.Writer writer = getWriter(); + Rows.writeClustering(clustering, writer); + + for (int i = 0; i < rows.length; i++) + { + if (rows[i] == null) + continue; + + rowInfo = rowInfo.mergeWith(rows[i].primaryKeyLivenessInfo()); + + if (rows[i].deletion().supersedes(rowDeletion)) + rowDeletion = rows[i].deletion(); + } + + if (rowDeletion.supersedes(activeDeletion)) + activeDeletion = rowDeletion; + + if (activeDeletion.deletes(rowInfo)) + rowInfo = LivenessInfo.NONE; + + writer.writePartitionKeyLivenessInfo(rowInfo); + writer.writeRowDeletion(rowDeletion); + + for (int i = 0; i < columns.simpleColumnCount(); i++) + { + ColumnDefinition c = columns.getSimple(i); + for (int j = 0; j < rows.length; j++) + cells[j] = rows[j] == null ? null : rows[j].getCell(c); + + reconcileCells(activeDeletion, writer); + } + + complexReducer.activeDeletion = activeDeletion; + complexReducer.writer = writer; + for (int i = 0; i < columns.complexColumnCount(); i++) + { + ColumnDefinition c = columns.getComplex(i); + + DeletionTime maxComplexDeletion = DeletionTime.LIVE; + for (int j = 0; j < rows.length; j++) + { + if (rows[j] == null) + continue; + + DeletionTime dt = rows[j].getDeletion(c); + if (complexDelTimes != null) + complexDelTimes[j] = dt; + + if (dt.supersedes(maxComplexDeletion)) + maxComplexDeletion = dt; + } + + boolean overrideActive = maxComplexDeletion.supersedes(activeDeletion); + maxComplexDeletion = overrideActive ? maxComplexDeletion : DeletionTime.LIVE; + writer.writeComplexDeletion(c, maxComplexDeletion); + if (listener != null) + listener.onMergedComplexDeletion(c, maxComplexDeletion, complexDelTimes); + + mergeComplex(overrideActive ? maxComplexDeletion : activeDeletion, c); + } + writer.endOfRow(); + + Row row = getRow(); + // Because shadowed cells are skipped, the row could be empty. In which case + // we return null (we also don't want to signal anything in that case since that + // means everything in the row was shadowed and the listener will have been signalled + // for whatever shadows it). + if (row.isEmpty()) + return null; + + maybeSignalEndOfRow(); + return row; + } + + private void maybeSignalListenerForRow() + { + if (listener != null && !signaledListenerForRow) + { + listener.onMergingRows(clustering, rowInfo, rowDeletion, rows); + signaledListenerForRow = true; + } + } + + private void maybeSignalListenerForCell(Cell merged, Cell[] versions) + { + if (listener != null) + { + maybeSignalListenerForRow(); + listener.onMergedCells(merged, versions); + } + } + + private void maybeSignalEndOfRow() + { + if (listener != null) + { + // If we haven't signaled the listener yet (we had no cells but some deletion info), do it now + maybeSignalListenerForRow(); + listener.onRowDone(); + } + } + + private void reconcileCells(DeletionTime activeDeletion, Row.Writer writer) + { + Cell reconciled = null; + for (int j = 0; j < cells.length; j++) + { + Cell cell = cells[j]; + if (cell != null && !activeDeletion.deletes(cell.livenessInfo())) + reconciled = Cells.reconcile(reconciled, cell, nowInSec); + } + + if (reconciled != null) + { + reconciled.writeTo(writer); + maybeSignalListenerForCell(reconciled, cells); + } + } + + private void mergeComplex(DeletionTime activeDeletion, ColumnDefinition c) + { + complexCells.clear(); + for (int j = 0; j < rows.length; j++) + { + Row row = rows[j]; + Iterator iter = row == null ? null : row.getCells(c); + complexCells.add(iter == null ? Iterators.emptyIterator() : iter); + } + + complexReducer.column = c; + complexReducer.activeDeletion = activeDeletion; + + // Note that we use the mergeIterator only to group cells to merge, but we + // write the result to the writer directly in the reducer, so all we care + // about is iterating over the result. + Iterator iter = MergeIterator.get(complexCells, c.cellComparator(), complexReducer); + while (iter.hasNext()) + iter.next(); + } + + private class ComplexColumnReducer extends MergeIterator.Reducer + { + private DeletionTime activeDeletion; + private Row.Writer writer; + private ColumnDefinition column; + + public void reduce(int idx, Cell current) + { + cells[idx] = current; + } + + protected Void getReduced() + { + reconcileCells(activeDeletion, writer); + return null; + } + + protected void onKeyChange() + { + Arrays.fill(cells, null); + } + } + + private static class StaticMerger extends Merger + { + private final StaticRow.Builder builder; + + private StaticMerger(CFMetaData metadata, int size, int nowInSec, Columns columns, UnfilteredRowIterators.MergeListener listener) + { + super(metadata, size, nowInSec, columns, listener); + this.builder = StaticRow.builder(columns, true, metadata.isCounter()); + } + + protected Row.Writer getWriter() + { + return builder; + } + + protected Row getRow() + { + return builder.build(); + } + } + + private static class RegularMerger extends Merger + { + private final ReusableRow row; + + private RegularMerger(CFMetaData metadata, int size, int nowInSec, Columns columns, UnfilteredRowIterators.MergeListener listener) + { + super(metadata, size, nowInSec, columns, listener); + this.row = new ReusableRow(metadata.clusteringColumns().size(), columns, true, metadata.isCounter()); + } + + protected Row.Writer getWriter() + { + return row.writer(); + } + + protected Row getRow() + { + return row; + } + } + } +} diff --git a/src/java/org/apache/cassandra/db/rows/RowAndTombstoneMergeIterator.java b/src/java/org/apache/cassandra/db/rows/RowAndTombstoneMergeIterator.java new file mode 100644 index 0000000000..51383a2253 --- /dev/null +++ b/src/java/org/apache/cassandra/db/rows/RowAndTombstoneMergeIterator.java @@ -0,0 +1,170 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.cassandra.db.rows; + +import java.util.Comparator; +import java.util.Iterator; + +import com.google.common.collect.PeekingIterator; +import com.google.common.collect.UnmodifiableIterator; + +import org.apache.cassandra.db.*; + +public class RowAndTombstoneMergeIterator extends UnmodifiableIterator implements PeekingIterator +{ + private final ClusteringComparator clusteringComparator; + private final Comparator comparator; + private final boolean reversed; + + private Iterator rowIter; + private Row nextRow; + + private Iterator tombstoneIter; + private RangeTombstone nextTombstone; + private boolean inTombstone; + + private Unfiltered next; + + public RowAndTombstoneMergeIterator(ClusteringComparator comparator, boolean reversed) + { + this.clusteringComparator = comparator; + this.comparator = reversed ? comparator.reversed() : comparator; + this.reversed = reversed; + } + + public RowAndTombstoneMergeIterator setTo(Iterator rowIter, Iterator tombstoneIter) + { + this.rowIter = rowIter; + this.tombstoneIter = tombstoneIter; + this.nextRow = null; + this.nextTombstone = null; + this.next = null; + this.inTombstone = false; + return this; + } + + public boolean isSet() + { + return rowIter != null; + } + + private void prepareNext() + { + if (next != null) + return; + + if (nextTombstone == null && tombstoneIter.hasNext()) + nextTombstone = tombstoneIter.next(); + if (nextRow == null && rowIter.hasNext()) + nextRow = rowIter.next(); + + if (nextTombstone == null) + { + if (nextRow == null) + return; + + next = nextRow; + nextRow = null; + } + else if (nextRow == null) + { + if (inTombstone) + { + RangeTombstone rt = nextTombstone; + nextTombstone = tombstoneIter.hasNext() ? tombstoneIter.next() : null; + if (nextTombstone != null && RangeTombstoneBoundaryMarker.isBoundary(clusteringComparator, rt.deletedSlice().close(reversed), nextTombstone.deletedSlice().open(reversed))) + { + next = RangeTombstoneBoundaryMarker.makeBoundary(reversed, + rt.deletedSlice().close(reversed), + nextTombstone.deletedSlice().open(reversed), + rt.deletionTime(), + nextTombstone.deletionTime()); + } + else + { + inTombstone = false; + next = new RangeTombstoneBoundMarker(rt.deletedSlice().close(reversed), rt.deletionTime()); + } + } + else + { + inTombstone = true; + next = new RangeTombstoneBoundMarker(nextTombstone.deletedSlice().open(reversed), nextTombstone.deletionTime()); + } + } + else if (inTombstone) + { + if (comparator.compare(nextTombstone.deletedSlice().close(reversed), nextRow.clustering()) < 0) + { + RangeTombstone rt = nextTombstone; + nextTombstone = tombstoneIter.hasNext() ? tombstoneIter.next() : null; + if (nextTombstone != null && RangeTombstoneBoundaryMarker.isBoundary(clusteringComparator, rt.deletedSlice().close(reversed), nextTombstone.deletedSlice().open(reversed))) + { + next = RangeTombstoneBoundaryMarker.makeBoundary(reversed, + rt.deletedSlice().close(reversed), + nextTombstone.deletedSlice().open(reversed), + rt.deletionTime(), + nextTombstone.deletionTime()); + } + else + { + inTombstone = false; + next = new RangeTombstoneBoundMarker(rt.deletedSlice().close(reversed), rt.deletionTime()); + } + } + else + { + next = nextRow; + nextRow = null; + } + } + else + { + if (comparator.compare(nextTombstone.deletedSlice().open(reversed), nextRow.clustering()) < 0) + { + inTombstone = true; + next = new RangeTombstoneBoundMarker(nextTombstone.deletedSlice().open(reversed), nextTombstone.deletionTime()); + } + else + { + next = nextRow; + nextRow = null; + } + } + } + + public boolean hasNext() + { + prepareNext(); + return next != null; + } + + public Unfiltered next() + { + prepareNext(); + Unfiltered toReturn = next; + next = null; + return toReturn; + } + + public Unfiltered peek() + { + prepareNext(); + return next(); + } +} diff --git a/src/java/org/apache/cassandra/db/rows/RowDataBlock.java b/src/java/org/apache/cassandra/db/rows/RowDataBlock.java new file mode 100644 index 0000000000..b1e2b13d04 --- /dev/null +++ b/src/java/org/apache/cassandra/db/rows/RowDataBlock.java @@ -0,0 +1,275 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.cassandra.db.rows; + +import java.nio.ByteBuffer; +import java.util.*; + +import com.google.common.collect.UnmodifiableIterator; + +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import org.apache.cassandra.config.ColumnDefinition; +import org.apache.cassandra.db.*; +import org.apache.cassandra.utils.ObjectSizes; + +/** + * A {@code RowDataBlock} holds data for one or more row (of a given table). More precisely, it contains + * cell data and complex deletion data (for complex columns) and allow access to this data. Please note + * however that {@code RowDataBlock} only holds the data inside the row, it does not hold the data + * pertaining to the row itself: clustering, partition key liveness info and row deletion. + *

+ * {@code RowDataBlock} is largely an implementation detail: it is only there to be reused by + * {@link AbstractPartitionData} and every concrete row implementation. + */ +public class RowDataBlock +{ + private static final Logger logger = LoggerFactory.getLogger(RowDataBlock.class); + + private static final long EMPTY_SIZE = ObjectSizes.measure(new RowDataBlock(Columns.NONE, 0, false, false)); + + // We distinguish 2 sub-objects: SimpleRowDataBlock that contains the data for the simple columns only, + // and ComplexRowDataBlock that only contains data for complex columns. The reason for having 2 separate + // objects is that simple columns are much easier to handle since we have only a single cell per-object + // and thus having a more specialized object allow a simpler and more efficient handling. + final SimpleRowDataBlock simpleData; + final ComplexRowDataBlock complexData; + + public RowDataBlock(Columns columns, int rows, boolean sortable, boolean isCounter) + { + this.simpleData = columns.hasSimple() ? new SimpleRowDataBlock(columns, rows, isCounter) : null; + this.complexData = columns.hasComplex() ? ComplexRowDataBlock.create(columns, rows, sortable, isCounter) : null; + } + + public Columns columns() + { + if (simpleData != null) + return simpleData.columns(); + if (complexData != null) + return complexData.columns(); + return Columns.NONE; + } + + /** + * Return the cell value for a given column of a given row. + * + * @param row the row for which to return the cell value. + * @param column the column for which to return the cell value. + * @param path the cell path for which to return the cell value. Can be null for + * simple columns. + * + * @return the value of the cell of path {@code path} for {@code column} in row {@code row}, or + * {@code null} if their is no such cell. + */ + public ByteBuffer getValue(int row, ColumnDefinition column, CellPath path) + { + if (column.isComplex()) + { + return complexData.getValue(row, column, path); + } + else + { + int idx = columns().simpleIdx(column, 0); + assert idx >= 0; + return simpleData.data.value((row * columns().simpleColumnCount()) + idx); + } + } + + /** + * Sets the cell value for a given simple column of a given row. + * + * @param row the row for which to set the cell value. + * @param column the simple column for which to set the cell value. + * @param path the cell path for which to return the cell value. Can be null for + * simple columns. + * @param value the value to set. + */ + public void setValue(int row, ColumnDefinition column, CellPath path, ByteBuffer value) + { + if (column.isComplex()) + { + complexData.setValue(row, column, path, value); + } + else + { + int idx = columns().simpleIdx(column, 0); + assert idx >= 0; + simpleData.data.setValue((row * columns().simpleColumnCount()) + idx, value); + } + } + + public static ReusableIterator reusableIterator() + { + return new ReusableIterator(); + } + + // Swap row i and j + public void swap(int i, int j) + { + if (simpleData != null) + simpleData.swap(i, j); + if (complexData != null) + complexData.swap(i, j); + } + + // Merge row i into j + public void merge(int i, int j, int nowInSec) + { + if (simpleData != null) + simpleData.merge(i, j, nowInSec); + if (complexData != null) + complexData.merge(i, j, nowInSec); + } + + // Move row i into j + public void move(int i, int j) + { + if (simpleData != null) + simpleData.move(i, j); + if (complexData != null) + complexData.move(i, j); + } + + public boolean hasComplexDeletion(int row) + { + return complexData != null && complexData.hasComplexDeletion(row); + } + + public long unsharedHeapSizeExcludingData() + { + return EMPTY_SIZE + + (simpleData == null ? 0 : simpleData.unsharedHeapSizeExcludingData()) + + (complexData == null ? 0 : complexData.unsharedHeapSizeExcludingData()); + } + + public static int computeNewCapacity(int currentCapacity, int idxToSet) + { + int newCapacity = currentCapacity == 0 ? 4 : currentCapacity; + while (idxToSet >= newCapacity) + newCapacity = 1 + (newCapacity * 3) / 2; + return newCapacity; + } + + public int dataSize() + { + return (simpleData == null ? 0 : simpleData.dataSize()) + + (complexData == null ? 0 : complexData.dataSize()); + } + + public void clear() + { + if (simpleData != null) + simpleData.clear(); + if (complexData != null) + complexData.clear(); + } + + public abstract static class Writer implements Row.Writer + { + private final boolean inOrderCells; + + protected int row; + + protected SimpleRowDataBlock.CellWriter simpleWriter; + protected ComplexRowDataBlock.CellWriter complexWriter; + + protected Writer(boolean inOrderCells) + { + this.inOrderCells = inOrderCells; + } + + protected Writer(RowDataBlock data, boolean inOrderCells) + { + this(inOrderCells); + updateWriter(data); + } + + protected void updateWriter(RowDataBlock data) + { + this.simpleWriter = data.simpleData == null ? null : data.simpleData.cellWriter(inOrderCells); + this.complexWriter = data.complexData == null ? null : data.complexData.cellWriter(inOrderCells); + } + + public Writer reset() + { + row = 0; + + if (simpleWriter != null) + simpleWriter.reset(); + if (complexWriter != null) + complexWriter.reset(); + + return this; + } + + public void writeCell(ColumnDefinition column, boolean isCounter, ByteBuffer value, LivenessInfo info, CellPath path) + { + if (column.isComplex()) + complexWriter.addCell(column, value, info, path); + else + simpleWriter.addCell(column, value, info); + } + + public void writeComplexDeletion(ColumnDefinition c, DeletionTime complexDeletion) + { + if (complexDeletion.isLive()) + return; + + complexWriter.setComplexDeletion(c, complexDeletion); + } + + public void endOfRow() + { + ++row; + if (simpleWriter != null) + simpleWriter.endOfRow(); + if (complexWriter != null) + complexWriter.endOfRow(); + } + } + + static class ReusableIterator extends UnmodifiableIterator implements Iterator + { + private SimpleRowDataBlock.ReusableIterator simpleIterator; + private ComplexRowDataBlock.ReusableIterator complexIterator; + + public ReusableIterator() + { + this.simpleIterator = SimpleRowDataBlock.reusableIterator(); + this.complexIterator = ComplexRowDataBlock.reusableIterator(); + } + + public ReusableIterator setTo(RowDataBlock dataBlock, int row) + { + simpleIterator.setTo(dataBlock.simpleData, row); + complexIterator.setTo(dataBlock.complexData, row); + return this; + } + + public boolean hasNext() + { + return simpleIterator.hasNext() || complexIterator.hasNext(); + } + + public Cell next() + { + return simpleIterator.hasNext() ? simpleIterator.next() : complexIterator.next(); + } + } +} diff --git a/src/java/org/apache/cassandra/db/rows/RowIterator.java b/src/java/org/apache/cassandra/db/rows/RowIterator.java new file mode 100644 index 0000000000..69994ddafb --- /dev/null +++ b/src/java/org/apache/cassandra/db/rows/RowIterator.java @@ -0,0 +1,76 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.cassandra.db.rows; + +import java.util.Iterator; + +import org.apache.cassandra.config.CFMetaData; +import org.apache.cassandra.db.*; + +/** + * An iterator over rows belonging to a partition. + * + * A RowIterator is an UnfilteredRowIterator to which any deletion information has been + * filtered out. As such, all cell of all rows returned by this iterator are, + * by definition, live, and hence code using a RowIterator don't have to worry + * about tombstones and other deletion information. + * + * Note that as for UnfilteredRowIterator, the rows returned must be in clustering order (or + * reverse clustering order if isReverseOrder is true), and the Row objects returned + * by next() are only valid until the next call to hasNext() or next(). + */ +public interface RowIterator extends Iterator, AutoCloseable +{ + /** + * The metadata for the table this iterator on. + */ + public CFMetaData metadata(); + + /** + * Whether or not the rows returned by this iterator are in reversed + * clustering order. + */ + public boolean isReverseOrder(); + + /** + * A subset of the columns for the (static and regular) rows returned by this iterator. + * Every row returned by this iterator must guarantee that it has only those columns. + */ + public PartitionColumns columns(); + + /** + * The partition key of the partition this in an iterator over. + */ + public DecoratedKey partitionKey(); + + /** + * The static part corresponding to this partition (this can be an empty + * row). + */ + public Row staticRow(); + + public void close(); + + /** + * Returns whether the provided iterator has no data. + */ + public default boolean isEmpty() + { + return staticRow().isEmpty() && !hasNext(); + } +} diff --git a/src/java/org/apache/cassandra/db/rows/RowIterators.java b/src/java/org/apache/cassandra/db/rows/RowIterators.java new file mode 100644 index 0000000000..a3bd91377e --- /dev/null +++ b/src/java/org/apache/cassandra/db/rows/RowIterators.java @@ -0,0 +1,152 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.cassandra.db.rows; + +import java.util.*; +import java.security.MessageDigest; + +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import org.apache.cassandra.config.CFMetaData; +import org.apache.cassandra.db.*; +import org.apache.cassandra.db.partitions.PartitionUpdate; +import org.apache.cassandra.utils.FBUtilities; + +/** + * Static methods to work with row iterators. + */ +public abstract class RowIterators +{ + private static final Logger logger = LoggerFactory.getLogger(RowIterators.class); + + private RowIterators() {} + + public static PartitionUpdate toUpdate(RowIterator iterator) + { + PartitionUpdate update = new PartitionUpdate(iterator.metadata(), iterator.partitionKey(), iterator.columns(), 1); + + if (iterator.staticRow() != Rows.EMPTY_STATIC_ROW) + iterator.staticRow().copyTo(update.staticWriter()); + + while (iterator.hasNext()) + iterator.next().copyTo(update.writer()); + + return update; + } + + public static void digest(RowIterator iterator, MessageDigest digest) + { + // TODO: we're not computing digest the same way that old nodes so we'll need + // to pass the version we're computing the digest for and deal with that. + digest.update(iterator.partitionKey().getKey().duplicate()); + iterator.columns().digest(digest); + FBUtilities.updateWithBoolean(digest, iterator.isReverseOrder()); + iterator.staticRow().digest(digest); + + while (iterator.hasNext()) + iterator.next().digest(digest); + } + + public static RowIterator emptyIterator(final CFMetaData cfm, final DecoratedKey partitionKey, final boolean isReverseOrder) + { + return new RowIterator() + { + public CFMetaData metadata() + { + return cfm; + } + + public boolean isReverseOrder() + { + return isReverseOrder; + } + + public PartitionColumns columns() + { + return PartitionColumns.NONE; + } + + public DecoratedKey partitionKey() + { + return partitionKey; + } + + public Row staticRow() + { + return Rows.EMPTY_STATIC_ROW; + } + + public boolean hasNext() + { + return false; + } + + public Row next() + { + throw new NoSuchElementException(); + } + + public void remove() + { + throw new UnsupportedOperationException(); + } + + public void close() + { + } + }; + } + + /** + * Wraps the provided iterator so it logs the returned rows for debugging purposes. + *

+ * Note that this is only meant for debugging as this can log a very large amount of + * logging at INFO. + */ + public static RowIterator loggingIterator(RowIterator iterator, final String id) + { + CFMetaData metadata = iterator.metadata(); + logger.info("[{}] Logging iterator on {}.{}, partition key={}, reversed={}", + new Object[]{ id, + metadata.ksName, + metadata.cfName, + metadata.getKeyValidator().getString(iterator.partitionKey().getKey()), + iterator.isReverseOrder() }); + + return new WrappingRowIterator(iterator) + { + @Override + public Row staticRow() + { + Row row = super.staticRow(); + if (!row.isEmpty()) + logger.info("[{}] {}", id, row.toString(metadata())); + return row; + } + + @Override + public Row next() + { + Row next = super.next(); + logger.info("[{}] {}", id, next.toString(metadata())); + return next; + } + }; + } +} diff --git a/src/java/org/apache/cassandra/db/rows/RowStats.java b/src/java/org/apache/cassandra/db/rows/RowStats.java new file mode 100644 index 0000000000..1bffdbeea2 --- /dev/null +++ b/src/java/org/apache/cassandra/db/rows/RowStats.java @@ -0,0 +1,237 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.cassandra.db.rows; + +import java.io.DataInput; +import java.io.IOException; +import java.util.Objects; + +import org.apache.cassandra.db.DeletionTime; +import org.apache.cassandra.db.TypeSizes; +import org.apache.cassandra.io.util.DataOutputPlus; + +import static org.apache.cassandra.db.LivenessInfo.NO_TIMESTAMP; +import static org.apache.cassandra.db.LivenessInfo.NO_TTL; +import static org.apache.cassandra.db.LivenessInfo.NO_DELETION_TIME; + +/** + * General statistics on rows (and and tombstones) for a given source. + *

+ * Those stats are used to optimize the on-wire and on-disk storage of rows. More precisely, + * the {@code minTimestamp}, {@code minLocalDeletionTime} and {@code minTTL} stats are used to + * delta-encode those information for the sake of vint encoding. And {@code avgColumnSetPerRow} + * is used to decide if cells should be stored in a sparse or dense way (see {@link UnfilteredSerializer}). + *

+ * Note that due to their use, those stats can suffer to be somewhat inaccurate (the more incurrate + * they are, the less effective the storage will be, but provided the stats are not completly wacky, + * this shouldn't have too huge an impact on performance) and in fact they will not always be + * accurate for reasons explained in {@link SerializationHeader#make}. + */ +public class RowStats +{ + // We should use this sparingly obviously + public static final RowStats NO_STATS = new RowStats(NO_TIMESTAMP, NO_DELETION_TIME, NO_TTL, -1); + + public static final Serializer serializer = new Serializer(); + + public final long minTimestamp; + public final int minLocalDeletionTime; + public final int minTTL; + + // Will be < 0 if the value is unknown + public final int avgColumnSetPerRow; + + public RowStats(long minTimestamp, + int minLocalDeletionTime, + int minTTL, + int avgColumnSetPerRow) + { + this.minTimestamp = minTimestamp; + this.minLocalDeletionTime = minLocalDeletionTime; + this.minTTL = minTTL; + this.avgColumnSetPerRow = avgColumnSetPerRow; + } + + public boolean hasMinTimestamp() + { + return minTimestamp != NO_TIMESTAMP; + } + + public boolean hasMinLocalDeletionTime() + { + return minLocalDeletionTime != NO_DELETION_TIME; + } + + /** + * Merge this stats with another one. + *

+ * The comments of {@link SerializationHeader#make} applies here too, i.e. the result of + * merging will be not totally accurate but we can live with that. + */ + public RowStats mergeWith(RowStats that) + { + long minTimestamp = this.minTimestamp == NO_TIMESTAMP + ? that.minTimestamp + : (that.minTimestamp == NO_TIMESTAMP ? this.minTimestamp : Math.min(this.minTimestamp, that.minTimestamp)); + + int minDelTime = this.minLocalDeletionTime == NO_DELETION_TIME + ? that.minLocalDeletionTime + : (that.minLocalDeletionTime == NO_DELETION_TIME ? this.minLocalDeletionTime : Math.min(this.minLocalDeletionTime, that.minLocalDeletionTime)); + + int minTTL = this.minTTL == NO_TTL + ? that.minTTL + : (that.minTTL == NO_TTL ? this.minTTL : Math.min(this.minTTL, that.minTTL)); + + int avgColumnSetPerRow = this.avgColumnSetPerRow < 0 + ? that.avgColumnSetPerRow + : (that.avgColumnSetPerRow < 0 ? this.avgColumnSetPerRow : (this.avgColumnSetPerRow + that.avgColumnSetPerRow) / 2); + + return new RowStats(minTimestamp, minDelTime, minTTL, avgColumnSetPerRow); + } + + @Override + public boolean equals(Object o) + { + if (this == o) return true; + if (o == null || getClass() != o.getClass()) return false; + + RowStats rowStats = (RowStats) o; + + if (avgColumnSetPerRow != rowStats.avgColumnSetPerRow) return false; + if (minLocalDeletionTime != rowStats.minLocalDeletionTime) return false; + if (minTTL != rowStats.minTTL) return false; + if (minTimestamp != rowStats.minTimestamp) return false; + + return true; + } + + @Override + public int hashCode() + { + return Objects.hash(minTimestamp, minLocalDeletionTime, minTTL, avgColumnSetPerRow); + } + + @Override + public String toString() + { + return String.format("RowStats(ts=%d, ldt=%d, ttl=%d, avgColPerRow=%d)", minTimestamp, minLocalDeletionTime, minTTL, avgColumnSetPerRow); + } + + public static class Collector + { + private boolean isTimestampSet; + private long minTimestamp = Long.MAX_VALUE; + + private boolean isDelTimeSet; + private int minDeletionTime = Integer.MAX_VALUE; + + private boolean isTTLSet; + private int minTTL = Integer.MAX_VALUE; + + private boolean isColumnSetPerRowSet; + private long totalColumnsSet; + private long rows; + + public void updateTimestamp(long timestamp) + { + if (timestamp == NO_TIMESTAMP) + return; + + isTimestampSet = true; + minTimestamp = Math.min(minTimestamp, timestamp); + } + + public void updateLocalDeletionTime(int deletionTime) + { + if (deletionTime == NO_DELETION_TIME) + return; + + isDelTimeSet = true; + minDeletionTime = Math.min(minDeletionTime, deletionTime); + } + + public void updateDeletionTime(DeletionTime deletionTime) + { + if (deletionTime.isLive()) + return; + + updateTimestamp(deletionTime.markedForDeleteAt()); + updateLocalDeletionTime(deletionTime.localDeletionTime()); + } + + public void updateTTL(int ttl) + { + if (ttl <= NO_TTL) + return; + + isTTLSet = true; + minTTL = Math.min(minTTL, ttl); + } + + public void updateColumnSetPerRow(int columnSetInRow) + { + updateColumnSetPerRow(columnSetInRow, 1); + } + + public void updateColumnSetPerRow(long totalColumnsSet, long rows) + { + if (totalColumnsSet < 0 || rows < 0) + return; + + this.isColumnSetPerRowSet = true; + this.totalColumnsSet += totalColumnsSet; + this.rows += rows; + } + + public RowStats get() + { + return new RowStats(isTimestampSet ? minTimestamp : NO_TIMESTAMP, + isDelTimeSet ? minDeletionTime : NO_DELETION_TIME, + isTTLSet ? minTTL : NO_TTL, + isColumnSetPerRowSet ? (rows == 0 ? 0 : (int)(totalColumnsSet / rows)) : -1); + } + } + + public static class Serializer + { + public void serialize(RowStats stats, DataOutputPlus out) throws IOException + { + out.writeLong(stats.minTimestamp); + out.writeInt(stats.minLocalDeletionTime); + out.writeInt(stats.minTTL); + out.writeInt(stats.avgColumnSetPerRow); + } + + public int serializedSize(RowStats stats, TypeSizes sizes) + { + return sizes.sizeof(stats.minTimestamp) + + sizes.sizeof(stats.minLocalDeletionTime) + + sizes.sizeof(stats.minTTL) + + sizes.sizeof(stats.avgColumnSetPerRow); + } + + public RowStats deserialize(DataInput in) throws IOException + { + long minTimestamp = in.readLong(); + int minLocalDeletionTime = in.readInt(); + int minTTL = in.readInt(); + int avgColumnSetPerRow = in.readInt(); + return new RowStats(minTimestamp, minLocalDeletionTime, minTTL, avgColumnSetPerRow); + } + } +} diff --git a/src/java/org/apache/cassandra/db/rows/Rows.java b/src/java/org/apache/cassandra/db/rows/Rows.java new file mode 100644 index 0000000000..76dcf60be4 --- /dev/null +++ b/src/java/org/apache/cassandra/db/rows/Rows.java @@ -0,0 +1,205 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.cassandra.db.rows; + +import java.util.*; + +import com.google.common.collect.Iterators; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import org.apache.cassandra.config.CFMetaData; +import org.apache.cassandra.config.ColumnDefinition; +import org.apache.cassandra.db.*; +import org.apache.cassandra.db.index.SecondaryIndexManager; +import org.apache.cassandra.utils.FBUtilities; +import org.apache.cassandra.utils.SearchIterator; + +/** + * Static utilities to work on Row objects. + */ +public abstract class Rows +{ + private static final Logger logger = LoggerFactory.getLogger(Rows.class); + + private Rows() {} + + public static final Row EMPTY_STATIC_ROW = new AbstractRow() + { + public Columns columns() + { + return Columns.NONE; + } + + public LivenessInfo primaryKeyLivenessInfo() + { + return LivenessInfo.NONE; + } + + public DeletionTime deletion() + { + return DeletionTime.LIVE; + } + + public boolean isEmpty() + { + return true; + } + + public boolean hasComplexDeletion() + { + return false; + } + + public Clustering clustering() + { + return Clustering.STATIC_CLUSTERING; + } + + public Cell getCell(ColumnDefinition c) + { + return null; + } + + public Cell getCell(ColumnDefinition c, CellPath path) + { + return null; + } + + public Iterator getCells(ColumnDefinition c) + { + return null; + } + + public DeletionTime getDeletion(ColumnDefinition c) + { + return DeletionTime.LIVE; + } + + public Iterator iterator() + { + return Iterators.emptyIterator(); + } + + public SearchIterator searchIterator() + { + return new SearchIterator() + { + public boolean hasNext() + { + return false; + } + + public ColumnData next(ColumnDefinition column) + { + return null; + } + }; + } + + public Kind kind() + { + return Unfiltered.Kind.ROW; + } + + public Row takeAlias() + { + return this; + } + }; + + public interface SimpleMergeListener + { + public void onAdded(Cell newCell); + public void onRemoved(Cell removedCell); + public void onUpdated(Cell existingCell, Cell updatedCell); + } + + public static void writeClustering(Clustering clustering, Row.Writer writer) + { + for (int i = 0; i < clustering.size(); i++) + writer.writeClusteringValue(clustering.get(i)); + } + + public static void merge(Row row1, Row row2, Columns mergedColumns, Row.Writer writer, int nowInSec) + { + merge(row1, row2, mergedColumns, writer, nowInSec, SecondaryIndexManager.nullUpdater); + } + + // Merge rows in memtable + // Return the minimum timestamp delta between existing and update + public static long merge(Row existing, + Row update, + Columns mergedColumns, + Row.Writer writer, + int nowInSec, + SecondaryIndexManager.Updater indexUpdater) + { + Clustering clustering = existing.clustering(); + writeClustering(clustering, writer); + + LivenessInfo existingInfo = existing.primaryKeyLivenessInfo(); + LivenessInfo updateInfo = update.primaryKeyLivenessInfo(); + LivenessInfo mergedInfo = existingInfo.mergeWith(updateInfo); + + long timeDelta = Math.abs(existingInfo.timestamp() - mergedInfo.timestamp()); + + DeletionTime deletion = existing.deletion().supersedes(update.deletion()) ? existing.deletion() : update.deletion(); + + if (deletion.deletes(mergedInfo)) + mergedInfo = LivenessInfo.NONE; + + writer.writePartitionKeyLivenessInfo(mergedInfo); + writer.writeRowDeletion(deletion); + + indexUpdater.maybeIndex(clustering, mergedInfo.timestamp(), mergedInfo.ttl(), deletion); + + for (int i = 0; i < mergedColumns.simpleColumnCount(); i++) + { + ColumnDefinition c = mergedColumns.getSimple(i); + Cell existingCell = existing.getCell(c); + Cell updateCell = update.getCell(c); + timeDelta = Math.min(timeDelta, Cells.reconcile(clustering, + existingCell, + updateCell, + deletion, + writer, + nowInSec, + indexUpdater)); + } + + for (int i = 0; i < mergedColumns.complexColumnCount(); i++) + { + ColumnDefinition c = mergedColumns.getComplex(i); + DeletionTime existingDt = existing.getDeletion(c); + DeletionTime updateDt = update.getDeletion(c); + DeletionTime maxDt = existingDt.supersedes(updateDt) ? existingDt : updateDt; + if (maxDt.supersedes(deletion)) + writer.writeComplexDeletion(c, maxDt); + else + maxDt = deletion; + + Iterator existingCells = existing.getCells(c); + Iterator updateCells = update.getCells(c); + timeDelta = Math.min(timeDelta, Cells.reconcileComplex(clustering, c, existingCells, updateCells, maxDt, writer, nowInSec, indexUpdater)); + } + + writer.endOfRow(); + return timeDelta; + } +} diff --git a/src/java/org/apache/cassandra/db/rows/SerializationHelper.java b/src/java/org/apache/cassandra/db/rows/SerializationHelper.java new file mode 100644 index 0000000000..56b993cf04 --- /dev/null +++ b/src/java/org/apache/cassandra/db/rows/SerializationHelper.java @@ -0,0 +1,137 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.cassandra.db.rows; + +import java.nio.ByteBuffer; + +import org.apache.cassandra.config.ColumnDefinition; +import org.apache.cassandra.db.*; +import org.apache.cassandra.db.context.CounterContext; +import org.apache.cassandra.db.filter.ColumnFilter; +import org.apache.cassandra.utils.ByteBufferUtil; + +public class SerializationHelper +{ + /** + * Flag affecting deserialization behavior (this only affect counters in practice). + * - LOCAL: for deserialization of local data (Expired columns are + * converted to tombstones (to gain disk space)). + * - FROM_REMOTE: for deserialization of data received from remote hosts + * (Expired columns are converted to tombstone and counters have + * their delta cleared) + * - PRESERVE_SIZE: used when no transformation must be performed, i.e, + * when we must ensure that deserializing and reserializing the + * result yield the exact same bytes. Streaming uses this. + */ + public static enum Flag + { + LOCAL, FROM_REMOTE, PRESERVE_SIZE; + } + + private final Flag flag; + public final int version; + + private final ReusableLivenessInfo livenessInfo = new ReusableLivenessInfo(); + + // The currently read row liveness infos (timestamp, ttl and localDeletionTime). + private long rowTimestamp; + private int rowTTL; + private int rowLocalDeletionTime; + + private final ColumnFilter columnsToFetch; + private ColumnFilter.Tester tester; + + public SerializationHelper(int version, Flag flag, ColumnFilter columnsToFetch) + { + this.flag = flag; + this.version = version; + this.columnsToFetch = columnsToFetch; + } + + public SerializationHelper(int version, Flag flag) + { + this(version, flag, null); + } + + public void writePartitionKeyLivenessInfo(Row.Writer writer, long timestamp, int ttl, int localDeletionTime) + { + livenessInfo.setTo(timestamp, ttl, localDeletionTime); + writer.writePartitionKeyLivenessInfo(livenessInfo); + + rowTimestamp = timestamp; + rowTTL = ttl; + rowLocalDeletionTime = localDeletionTime; + } + + public long getRowTimestamp() + { + return rowTimestamp; + } + + public int getRowTTL() + { + return rowTTL; + } + + public int getRowLocalDeletionTime() + { + return rowLocalDeletionTime; + } + + public boolean includes(ColumnDefinition column) + { + return columnsToFetch == null || columnsToFetch.includes(column); + } + + public boolean canSkipValue(ColumnDefinition column) + { + return columnsToFetch != null && columnsToFetch.canSkipValue(column); + } + + public void startOfComplexColumn(ColumnDefinition column) + { + this.tester = columnsToFetch == null ? null : columnsToFetch.newTester(column); + } + + public void endOfComplexColumn(ColumnDefinition column) + { + this.tester = null; + } + + public void writeCell(Row.Writer writer, + ColumnDefinition column, + boolean isCounter, + ByteBuffer value, + long timestamp, + int localDelTime, + int ttl, + CellPath path) + { + livenessInfo.setTo(timestamp, ttl, localDelTime); + + if (isCounter && ((flag == Flag.FROM_REMOTE || (flag == Flag.LOCAL && CounterContext.instance().shouldClearLocal(value))))) + value = CounterContext.instance().clearAllLocal(value); + + if (!column.isComplex() || tester == null || tester.includes(path)) + { + if (tester != null && tester.canSkipValue(path)) + value = ByteBufferUtil.EMPTY_BYTE_BUFFER; + writer.writeCell(column, isCounter, value, livenessInfo, path); + } + } +} diff --git a/src/java/org/apache/cassandra/db/rows/SimpleRowDataBlock.java b/src/java/org/apache/cassandra/db/rows/SimpleRowDataBlock.java new file mode 100644 index 0000000000..08f37fdc6a --- /dev/null +++ b/src/java/org/apache/cassandra/db/rows/SimpleRowDataBlock.java @@ -0,0 +1,188 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.cassandra.db.rows; + +import java.nio.ByteBuffer; + +import com.google.common.collect.UnmodifiableIterator; + +import org.apache.cassandra.config.ColumnDefinition; +import org.apache.cassandra.db.*; +import org.apache.cassandra.utils.ObjectSizes; + +/** + * Holds cells data for the simple columns of one or more rows. + *

+ * In practice, a {@code SimpleRowDataBlock} contains a single {@code CellData} "array" and + * the (simple) columns for which the {@code SimplerowDataBlock} has data for. The cell for + * a row i and a column c is stored in the {@code CellData} at index 'i * index(c)'. + *

+ * This does mean that we store cells in a "dense" way: if column doesn't have a cell for a + * given row, the correspond index in the cell data array will simple have a {@code null} value. + * We might want to switch to a more sparse encoding in the future but we keep it simple for + * now (having a sparse encoding make things a tad more complex because we need to be able to + * swap the cells for 2 given rows as seen in ComplexRowDataBlock). + */ +public class SimpleRowDataBlock +{ + private static final long EMPTY_SIZE = ObjectSizes.measure(new SimpleRowDataBlock(Columns.NONE, 0, false)); + + final Columns columns; + final CellData data; + + public SimpleRowDataBlock(Columns columns, int rows, boolean isCounter) + { + this.columns = columns; + this.data = new CellData(rows * columns.simpleColumnCount(), isCounter); + } + + public Columns columns() + { + return columns; + } + + // Swap row i and j + public void swap(int i, int j) + { + int s = columns.simpleColumnCount(); + for (int k = 0; k < s; k++) + data.swapCell(i * s + k, j * s + k); + } + + // Merge row i into j + public void merge(int i, int j, int nowInSec) + { + int s = columns.simpleColumnCount(); + for (int k = 0; k < s; k++) + data.mergeCell(i * s + k, j * s + k, nowInSec); + } + + // Move row i into j + public void move(int i, int j) + { + int s = columns.simpleColumnCount(); + for (int k = 0; k < s; k++) + data.moveCell(i * s + k, j * s + k); + } + + public long unsharedHeapSizeExcludingData() + { + return EMPTY_SIZE + data.unsharedHeapSizeExcludingData(); + } + + public int dataSize() + { + return data.dataSize(); + } + + public CellWriter cellWriter(boolean inOrderCells) + { + return new CellWriter(inOrderCells); + } + + public static CellData.ReusableCell reusableCell() + { + return new CellData.ReusableCell(); + } + + public static ReusableIterator reusableIterator() + { + return new ReusableIterator(); + } + + public void clear() + { + data.clear(); + } + + static class ReusableIterator extends UnmodifiableIterator + { + private SimpleRowDataBlock dataBlock; + private final CellData.ReusableCell cell = new CellData.ReusableCell(); + + private int base; + private int column; + + private ReusableIterator() + { + } + + public ReusableIterator setTo(SimpleRowDataBlock dataBlock, int row) + { + this.dataBlock = dataBlock; + this.base = dataBlock == null ? -1 : row * dataBlock.columns.simpleColumnCount(); + this.column = 0; + return this; + } + + public boolean hasNext() + { + if (dataBlock == null) + return false; + + int columnCount = dataBlock.columns.simpleColumnCount(); + // iterate over column until we find one with data + while (column < columnCount && !dataBlock.data.hasCell(base + column)) + ++column; + + return column < columnCount; + } + + public Cell next() + { + cell.setTo(dataBlock.data, dataBlock.columns.getSimple(column), base + column); + ++column; + return cell; + } + } + + public class CellWriter + { + private final boolean inOrderCells; + + private int base; + private int lastColumnIdx; + + public CellWriter(boolean inOrderCells) + { + this.inOrderCells = inOrderCells; + } + + public void addCell(ColumnDefinition column, ByteBuffer value, LivenessInfo info) + { + int fromIdx = inOrderCells ? lastColumnIdx : 0; + lastColumnIdx = columns.simpleIdx(column, fromIdx); + assert lastColumnIdx >= 0 : "Cannot find column " + column.name + " in " + columns + " from " + fromIdx; + int idx = base + lastColumnIdx; + data.setCell(idx, value, info); + } + + public void reset() + { + base = 0; + lastColumnIdx = 0; + data.clear(); + } + + public void endOfRow() + { + base += columns.simpleColumnCount(); + lastColumnIdx = 0; + } + } +} diff --git a/src/java/org/apache/cassandra/db/rows/SliceableUnfilteredRowIterator.java b/src/java/org/apache/cassandra/db/rows/SliceableUnfilteredRowIterator.java new file mode 100644 index 0000000000..2250ee9318 --- /dev/null +++ b/src/java/org/apache/cassandra/db/rows/SliceableUnfilteredRowIterator.java @@ -0,0 +1,39 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.cassandra.db.rows; + +import java.util.Iterator; + +import org.apache.cassandra.db.Slice; + +public interface SliceableUnfilteredRowIterator extends UnfilteredRowIterator +{ + /** + * Move forward (resp. backward if isReverseOrder() is true for the iterator) in + * the iterator and return an iterator over the Unfiltered selected by the provided + * {@code slice}. + *

+ * Please note that successive calls to {@code slice} are allowed provided the + * slice are non overlapping and are passed in clustering (resp. reverse clustering) order. + * However, {@code slice} is allowed to leave the iterator in an unknown state and there + * is no guarantee over what a call to {@code hasNext} or {@code next} will yield after + * a call to {@code slice}. In other words, for a given iterator, you should either use + * {@code slice} or {@code hasNext/next} but not both. + */ + public Iterator slice(Slice slice); +} diff --git a/src/java/org/apache/cassandra/db/rows/StaticRow.java b/src/java/org/apache/cassandra/db/rows/StaticRow.java new file mode 100644 index 0000000000..2ad9fb4c85 --- /dev/null +++ b/src/java/org/apache/cassandra/db/rows/StaticRow.java @@ -0,0 +1,193 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.cassandra.db.rows; + +import java.nio.ByteBuffer; +import java.util.Iterator; + +import org.apache.cassandra.db.*; + +import org.apache.cassandra.config.ColumnDefinition; +import org.apache.cassandra.utils.SearchIterator; + +public class StaticRow extends AbstractRow +{ + private final DeletionTime deletion; + private final RowDataBlock data; + + private StaticRow(DeletionTime deletion, RowDataBlock data) + { + this.deletion = deletion.takeAlias(); + this.data = data; + } + + public Columns columns() + { + return data.columns(); + } + + public Cell getCell(ColumnDefinition c) + { + assert !c.isComplex(); + if (data.simpleData == null) + return null; + + int idx = columns().simpleIdx(c, 0); + if (idx < 0) + return null; + + return SimpleRowDataBlock.reusableCell().setTo(data.simpleData.data, c, idx); + } + + public Cell getCell(ColumnDefinition c, CellPath path) + { + assert c.isComplex(); + + ComplexRowDataBlock dataBlock = data.complexData; + if (dataBlock == null) + return null; + + int idx = dataBlock.cellIdx(0, c, path); + if (idx < 0) + return null; + + return SimpleRowDataBlock.reusableCell().setTo(dataBlock.cellData(0), c, idx); + } + + public Iterator getCells(ColumnDefinition c) + { + assert c.isComplex(); + return ComplexRowDataBlock.reusableComplexCells().setTo(data.complexData, 0, c); + } + + public boolean hasComplexDeletion() + { + return data.hasComplexDeletion(0); + } + + public DeletionTime getDeletion(ColumnDefinition c) + { + assert c.isComplex(); + if (data.complexData == null) + return DeletionTime.LIVE; + + int idx = data.complexData.complexDeletionIdx(0, c); + return idx < 0 + ? DeletionTime.LIVE + : ComplexRowDataBlock.complexDeletionCursor().setTo(data.complexData.complexDelTimes, idx); + } + + public Iterator iterator() + { + return RowDataBlock.reusableIterator().setTo(data, 0); + } + + public SearchIterator searchIterator() + { + return new SearchIterator() + { + private int simpleIdx = 0; + + public boolean hasNext() + { + // TODO: we can do better, but we expect users to no rely on this anyway + return true; + } + + public ColumnData next(ColumnDefinition column) + { + if (column.isComplex()) + { + // TODO: this is sub-optimal + + Iterator cells = getCells(column); + return cells == null ? null : new ColumnData(column, null, cells, getDeletion(column)); + } + else + { + simpleIdx = columns().simpleIdx(column, simpleIdx); + assert simpleIdx >= 0; + + Cell cell = SimpleRowDataBlock.reusableCell().setTo(data.simpleData.data, column, simpleIdx); + ++simpleIdx; + return cell == null ? null : new ColumnData(column, cell, null, null); + } + } + }; + } + + public Row takeAlias() + { + return this; + } + + public Clustering clustering() + { + return Clustering.STATIC_CLUSTERING; + } + + public LivenessInfo primaryKeyLivenessInfo() + { + return LivenessInfo.NONE; + } + + public DeletionTime deletion() + { + return deletion; + } + + public static Builder builder(Columns columns, boolean inOrderCells, boolean isCounter) + { + return new Builder(columns, inOrderCells, isCounter); + } + + public static class Builder extends RowDataBlock.Writer + { + private final RowDataBlock data; + private DeletionTime deletion = DeletionTime.LIVE; + + public Builder(Columns columns, boolean inOrderCells, boolean isCounter) + { + super(inOrderCells); + this.data = new RowDataBlock(columns, 1, false, isCounter); + updateWriter(data); + } + + public void writeClusteringValue(ByteBuffer buffer) + { + throw new UnsupportedOperationException(); + } + + public void writePartitionKeyLivenessInfo(LivenessInfo info) + { + // Static rows are special and don't really have an existence unless they have live cells, + // so we shouldn't have any partition key liveness info. + assert info.equals(LivenessInfo.NONE); + } + + public void writeRowDeletion(DeletionTime deletion) + { + this.deletion = deletion; + } + + public StaticRow build() + { + return new StaticRow(deletion, data); + } + } +} diff --git a/src/java/org/apache/cassandra/db/columniterator/IdentityQueryFilter.java b/src/java/org/apache/cassandra/db/rows/TombstoneFilteringRow.java similarity index 60% rename from src/java/org/apache/cassandra/db/columniterator/IdentityQueryFilter.java rename to src/java/org/apache/cassandra/db/rows/TombstoneFilteringRow.java index 7185eef452..a1c0ddcfd2 100644 --- a/src/java/org/apache/cassandra/db/columniterator/IdentityQueryFilter.java +++ b/src/java/org/apache/cassandra/db/rows/TombstoneFilteringRow.java @@ -15,23 +15,34 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -package org.apache.cassandra.db.columniterator; +package org.apache.cassandra.db.rows; -import org.apache.cassandra.db.composites.Composites; -import org.apache.cassandra.db.filter.SliceQueryFilter; +import org.apache.cassandra.config.ColumnDefinition; +import org.apache.cassandra.db.*; -public class IdentityQueryFilter extends SliceQueryFilter +public class TombstoneFilteringRow extends FilteringRow { - /** - * Will read entire CF into memory. Use with caution. - */ - public IdentityQueryFilter() + private final int nowInSec; + + public TombstoneFilteringRow(int nowInSec) { - super(Composites.EMPTY, Composites.EMPTY, false, Integer.MAX_VALUE); + this.nowInSec = nowInSec; } @Override - protected boolean respectTombstoneThresholds() + protected boolean include(DeletionTime dt) + { + return false; + } + + @Override + protected boolean include(Cell cell) + { + return cell.isLive(nowInSec); + } + + @Override + protected boolean include(ColumnDefinition c, DeletionTime dt) { return false; } diff --git a/src/java/org/apache/cassandra/db/rows/Unfiltered.java b/src/java/org/apache/cassandra/db/rows/Unfiltered.java new file mode 100644 index 0000000000..b1692e3a6c --- /dev/null +++ b/src/java/org/apache/cassandra/db/rows/Unfiltered.java @@ -0,0 +1,60 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.cassandra.db.rows; + +import java.security.MessageDigest; + +import org.apache.cassandra.config.CFMetaData; +import org.apache.cassandra.db.Clusterable; + +/** + * Unfiltered is the common class for the main constituent of an unfiltered partition. + *

+ * In practice, an Unfiltered is either a row or a range tombstone marker. Unfiltereds + * are uniquely identified by their clustering information and can be sorted according + * to those. + */ +public interface Unfiltered extends Clusterable +{ + public enum Kind { ROW, RANGE_TOMBSTONE_MARKER }; + + /** + * The kind of the atom: either row or range tombstone marker. + */ + public Kind kind(); + + /** + * Digest the atom using the provided {@code MessageDigest}. + * + * @param digest the {@code MessageDigest} to use. + */ + public void digest(MessageDigest digest); + + /** + * Validate the data of this atom. + * + * @param metadata the metadata for the table this atom is part of. + * @throws MarshalException if some of the data in this atom is + * invalid (some value is invalid for its column type, or some field + * is nonsensical). + */ + public void validateData(CFMetaData metadata); + + public String toString(CFMetaData metadata); + public String toString(CFMetaData metadata, boolean fullDetails); +} diff --git a/src/java/org/apache/cassandra/db/rows/UnfilteredRowIterator.java b/src/java/org/apache/cassandra/db/rows/UnfilteredRowIterator.java new file mode 100644 index 0000000000..a3ecf6d5d1 --- /dev/null +++ b/src/java/org/apache/cassandra/db/rows/UnfilteredRowIterator.java @@ -0,0 +1,102 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.cassandra.db.rows; + +import java.util.Iterator; + +import org.apache.cassandra.config.CFMetaData; +import org.apache.cassandra.db.*; + +/** + * An iterator over the rows of a given partition that also includes deletion informations. + *

+ * An {@code UnfilteredRowIterator} contains a few partition top-level informations and is an + * iterator of {@code Unfiltered}, that is of either {@code Row} or {@code RangeTombstoneMarker}. + * An implementation of {@code UnfilteredRowIterator} must provide the following + * guarantees: + * 1. the returned {@code Unfiltered} must be in clustering order, or in reverse clustering + * order iff {@link #isReverseOrder} returns true. + * 2. the iterator should not shadow its own data. That is, no deletion + * (partition level deletion, row deletion, range tombstone, complex + * deletion) should delete anything else returned by the iterator (cell, row, ...). + * 3. every "start" range tombstone marker should have a corresponding "end" marker, and no other + * marker should be in-between this start-end pair of marker. Note that due to the + * previous rule this means that between a "start" and a corresponding "end" marker there + * can only be rows that are not deleted by the markers. Also note that when iterating + * in reverse order, "end" markers are returned before their "start" counterpart (i.e. + * "start" and "end" are always in the sense of the clustering order). + * + * Note further that the objects returned by next() are only valid until the + * next call to hasNext() or next(). If a consumer wants to keep a reference on + * the returned objects for longer than the iteration, it must make a copy of + * it explicitly. + */ +public interface UnfilteredRowIterator extends Iterator, AutoCloseable +{ + /** + * The metadata for the table this iterator on. + */ + public CFMetaData metadata(); + + /** + * A subset of the columns for the (static and regular) rows returned by this iterator. + * Every row returned by this iterator must guarantee that it has only those columns. + */ + public PartitionColumns columns(); + + /** + * Whether or not the atom returned by this iterator are in reversed + * clustering order. + */ + public boolean isReverseOrder(); + + /** + * The partition key of the partition this in an iterator over. + */ + public DecoratedKey partitionKey(); + + /** + * The partition level deletion for the partition this iterate over. + */ + public DeletionTime partitionLevelDeletion(); + + /** + * The static part corresponding to this partition (this can be an empty + * row). + */ + public Row staticRow(); + + /** + * Return "statistics" about what is returned by this iterator. Those are used for + * performance reasons (for delta-encoding for instance) and code should not + * expect those to be exact. + */ + public RowStats stats(); + + public void close(); + + /** + * Returns whether this iterator has no data (including no deletion data). + */ + public default boolean isEmpty() + { + return partitionLevelDeletion().isLive() + && staticRow().isEmpty() + && !hasNext(); + } +} diff --git a/src/java/org/apache/cassandra/db/rows/UnfilteredRowIteratorSerializer.java b/src/java/org/apache/cassandra/db/rows/UnfilteredRowIteratorSerializer.java new file mode 100644 index 0000000000..13c09d451d --- /dev/null +++ b/src/java/org/apache/cassandra/db/rows/UnfilteredRowIteratorSerializer.java @@ -0,0 +1,306 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.cassandra.db.rows; + +import java.io.DataInput; +import java.io.IOException; +import java.io.IOError; + +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import org.apache.cassandra.config.CFMetaData; +import org.apache.cassandra.db.*; +import org.apache.cassandra.io.util.DataOutputPlus; +import org.apache.cassandra.service.StorageService; +import org.apache.cassandra.utils.ByteBufferUtil; +import org.apache.cassandra.utils.FBUtilities; + +/** + * Serialize/Deserialize an unfiltered row iterator. + * + * The serialization is composed of a header, follows by the rows and range tombstones of the iterator serialized + * until we read the end of the partition (see UnfilteredSerializer for details). The header itself + * is: + * [][] + * where: + * is the table cfid. + * is the partition key. + * contains bit flags. Each flag is set if it's corresponding bit is set. From rightmost + * bit to leftmost one, the flags are: + * - is empty: whether the iterator is empty. If so, nothing follows the + * - is reversed: whether the iterator is in reversed clustering order + * - has partition deletion: whether or not there is a following + * - has static row: whether or not there is a following + * - has row estimate: whether or not there is a following + * is the SerializationHeader. More precisely it's + * [] + * where: + * - is the base timestamp used for delta-encoding timestamps + * - is the base localDeletionTime used for delta-encoding local deletion times + * - is the base localDeletionTime used for delta-encoding ttls + * - is the static columns if a static row is present. It's + * the number of columns as an unsigned short, followed by the column names. + * - is the columns of the rows of the iterator. It's serialized as . + * is the deletion time for the partition (delta-encoded) + * is the static row for this partition as serialized by UnfilteredSerializer. + * is the (potentially estimated) number of rows serialized. This is only use for + * the purpose of some sizing on the receiving end and should not be relied upon too strongly. + * + * !!! Please note that the serialized value depends on the schema and as such should not be used as is if + * it might be deserialized after the schema as changed !!! + * TODO: we should add a flag to include the relevant metadata in the header for commit log etc..... + */ +public class UnfilteredRowIteratorSerializer +{ + protected static final Logger logger = LoggerFactory.getLogger(UnfilteredRowIteratorSerializer.class); + + private static final int IS_EMPTY = 0x01; + private static final int IS_REVERSED = 0x02; + private static final int HAS_PARTITION_DELETION = 0x04; + private static final int HAS_STATIC_ROW = 0x08; + private static final int HAS_ROW_ESTIMATE = 0x10; + + public static final UnfilteredRowIteratorSerializer serializer = new UnfilteredRowIteratorSerializer(); + + public void serialize(UnfilteredRowIterator iterator, DataOutputPlus out, int version) throws IOException + { + serialize(iterator, out, version, -1); + } + + public void serialize(UnfilteredRowIterator iterator, DataOutputPlus out, int version, int rowEstimate) throws IOException + { + SerializationHeader header = new SerializationHeader(iterator.metadata(), + iterator.columns(), + iterator.stats()); + serialize(iterator, out, header, version, rowEstimate); + } + + public void serialize(UnfilteredRowIterator iterator, DataOutputPlus out, SerializationHeader header, int version, int rowEstimate) throws IOException + { + CFMetaData.serializer.serialize(iterator.metadata(), out, version); + ByteBufferUtil.writeWithLength(iterator.partitionKey().getKey(), out); + + int flags = 0; + if (iterator.isReverseOrder()) + flags |= IS_REVERSED; + + if (iterator.isEmpty()) + { + out.writeByte((byte)(flags | IS_EMPTY)); + return; + } + + DeletionTime partitionDeletion = iterator.partitionLevelDeletion(); + if (!partitionDeletion.isLive()) + flags |= HAS_PARTITION_DELETION; + Row staticRow = iterator.staticRow(); + boolean hasStatic = staticRow != Rows.EMPTY_STATIC_ROW; + if (hasStatic) + flags |= HAS_STATIC_ROW; + + if (rowEstimate >= 0) + flags |= HAS_ROW_ESTIMATE; + + out.writeByte((byte)flags); + + SerializationHeader.serializer.serializeForMessaging(header, out, hasStatic); + + if (!partitionDeletion.isLive()) + writeDelTime(partitionDeletion, header, out); + + if (hasStatic) + UnfilteredSerializer.serializer.serialize(staticRow, header, out, version); + + if (rowEstimate >= 0) + out.writeInt(rowEstimate); + + while (iterator.hasNext()) + UnfilteredSerializer.serializer.serialize(iterator.next(), header, out, version); + UnfilteredSerializer.serializer.writeEndOfPartition(out); + } + + // Please note that this consume the iterator, and as such should not be called unless we have a simple way to + // recreate an iterator for both serialize and serializedSize, which is mostly only PartitionUpdate + public long serializedSize(UnfilteredRowIterator iterator, int version, int rowEstimate, TypeSizes sizes) + { + SerializationHeader header = new SerializationHeader(iterator.metadata(), + iterator.columns(), + iterator.stats()); + + assert rowEstimate >= 0; + + long size = CFMetaData.serializer.serializedSize(iterator.metadata(), version, sizes) + + sizes.sizeofWithLength(iterator.partitionKey().getKey()) + + 1; // flags + + if (iterator.isEmpty()) + return size; + + DeletionTime partitionDeletion = iterator.partitionLevelDeletion(); + Row staticRow = iterator.staticRow(); + boolean hasStatic = staticRow != Rows.EMPTY_STATIC_ROW; + + size += SerializationHeader.serializer.serializedSizeForMessaging(header, sizes, hasStatic); + + if (!partitionDeletion.isLive()) + size += delTimeSerializedSize(partitionDeletion, header, sizes); + + if (hasStatic) + size += UnfilteredSerializer.serializer.serializedSize(staticRow, header, version, sizes); + + if (rowEstimate >= 0) + size += sizes.sizeof(rowEstimate); + + while (iterator.hasNext()) + size += UnfilteredSerializer.serializer.serializedSize(iterator.next(), header, version, sizes); + size += UnfilteredSerializer.serializer.serializedSizeEndOfPartition(sizes); + + return size; + } + + public Header deserializeHeader(DataInput in, int version, SerializationHelper.Flag flag) throws IOException + { + CFMetaData metadata = CFMetaData.serializer.deserialize(in, version); + DecoratedKey key = StorageService.getPartitioner().decorateKey(ByteBufferUtil.readWithLength(in)); + int flags = in.readUnsignedByte(); + boolean isReversed = (flags & IS_REVERSED) != 0; + if ((flags & IS_EMPTY) != 0) + { + SerializationHeader sh = new SerializationHeader(metadata, PartitionColumns.NONE, RowStats.NO_STATS); + return new Header(sh, metadata, key, isReversed, true, null, null, 0); + } + + boolean hasPartitionDeletion = (flags & HAS_PARTITION_DELETION) != 0; + boolean hasStatic = (flags & HAS_STATIC_ROW) != 0; + boolean hasRowEstimate = (flags & HAS_ROW_ESTIMATE) != 0; + + SerializationHeader header = SerializationHeader.serializer.deserializeForMessaging(in, metadata, hasStatic); + + DeletionTime partitionDeletion = hasPartitionDeletion ? readDelTime(in, header) : DeletionTime.LIVE; + + Row staticRow = Rows.EMPTY_STATIC_ROW; + if (hasStatic) + staticRow = UnfilteredSerializer.serializer.deserializeStaticRow(in, header, new SerializationHelper(version, flag)); + + int rowEstimate = hasRowEstimate ? in.readInt() : -1; + return new Header(header, metadata, key, isReversed, false, partitionDeletion, staticRow, rowEstimate); + } + + public void deserialize(DataInput in, SerializationHelper helper, SerializationHeader header, Row.Writer rowWriter, RangeTombstoneMarker.Writer markerWriter) throws IOException + { + while (UnfilteredSerializer.serializer.deserialize(in, header, helper, rowWriter, markerWriter) != null); + } + + public UnfilteredRowIterator deserialize(final DataInput in, int version, SerializationHelper.Flag flag) throws IOException + { + final Header h = deserializeHeader(in, version, flag); + + if (h.isEmpty) + return UnfilteredRowIterators.emptyIterator(h.metadata, h.key, h.isReversed); + + final int clusteringSize = h.metadata.clusteringColumns().size(); + final SerializationHelper helper = new SerializationHelper(version, flag); + + return new AbstractUnfilteredRowIterator(h.metadata, h.key, h.partitionDeletion, h.sHeader.columns(), h.staticRow, h.isReversed, h.sHeader.stats()) + { + private final ReusableRow row = new ReusableRow(clusteringSize, h.sHeader.columns().regulars, true, h.metadata.isCounter()); + private final RangeTombstoneMarker.Builder markerBuilder = new RangeTombstoneMarker.Builder(clusteringSize); + + protected Unfiltered computeNext() + { + try + { + Unfiltered.Kind kind = UnfilteredSerializer.serializer.deserialize(in, h.sHeader, helper, row.writer(), markerBuilder.reset()); + if (kind == null) + return endOfData(); + + return kind == Unfiltered.Kind.ROW ? row : markerBuilder.build(); + } + catch (IOException e) + { + throw new IOError(e); + } + } + }; + } + + public static void writeDelTime(DeletionTime dt, SerializationHeader header, DataOutputPlus out) throws IOException + { + out.writeLong(header.encodeTimestamp(dt.markedForDeleteAt())); + out.writeInt(header.encodeDeletionTime(dt.localDeletionTime())); + } + + public static long delTimeSerializedSize(DeletionTime dt, SerializationHeader header, TypeSizes sizes) + { + return sizes.sizeof(header.encodeTimestamp(dt.markedForDeleteAt())) + + sizes.sizeof(header.encodeDeletionTime(dt.localDeletionTime())); + } + + public static DeletionTime readDelTime(DataInput in, SerializationHeader header) throws IOException + { + long markedAt = header.decodeTimestamp(in.readLong()); + int localDelTime = header.decodeDeletionTime(in.readInt()); + return new SimpleDeletionTime(markedAt, localDelTime); + } + + public static void skipDelTime(DataInput in, SerializationHeader header) throws IOException + { + // Note that since we might use VINT, we shouldn't assume the size of a long or an int + in.readLong(); + in.readInt(); + } + + public static class Header + { + public final SerializationHeader sHeader; + public final CFMetaData metadata; + public final DecoratedKey key; + public final boolean isReversed; + public final boolean isEmpty; + public final DeletionTime partitionDeletion; + public final Row staticRow; + public final int rowEstimate; // -1 if no estimate + + private Header(SerializationHeader sHeader, + CFMetaData metadata, + DecoratedKey key, + boolean isReversed, + boolean isEmpty, + DeletionTime partitionDeletion, + Row staticRow, + int rowEstimate) + { + this.sHeader = sHeader; + this.metadata = metadata; + this.key = key; + this.isReversed = isReversed; + this.isEmpty = isEmpty; + this.partitionDeletion = partitionDeletion; + this.staticRow = staticRow; + this.rowEstimate = rowEstimate; + } + + @Override + public String toString() + { + return String.format("{header=%s, table=%s.%s, key=%s, isReversed=%b, isEmpty=%b, del=%s, staticRow=%s, rowEstimate=%d}", + sHeader, metadata.ksName, metadata.cfName, key, isReversed, isEmpty, partitionDeletion, staticRow.toString(metadata), rowEstimate); + } + } +} diff --git a/src/java/org/apache/cassandra/db/rows/UnfilteredRowIterators.java b/src/java/org/apache/cassandra/db/rows/UnfilteredRowIterators.java new file mode 100644 index 0000000000..2c71cf3fe4 --- /dev/null +++ b/src/java/org/apache/cassandra/db/rows/UnfilteredRowIterators.java @@ -0,0 +1,770 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.cassandra.db.rows; + +import java.nio.ByteBuffer; +import java.util.*; +import java.security.MessageDigest; + +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; +import com.google.common.collect.AbstractIterator; + +import org.apache.cassandra.config.CFMetaData; +import org.apache.cassandra.config.ColumnDefinition; +import org.apache.cassandra.db.*; +import org.apache.cassandra.db.partitions.PartitionUpdate; +import org.apache.cassandra.io.util.FileUtils; +import org.apache.cassandra.io.sstable.CorruptSSTableException; +import org.apache.cassandra.serializers.MarshalException; +import org.apache.cassandra.utils.ByteBufferUtil; +import org.apache.cassandra.utils.FBUtilities; +import org.apache.cassandra.utils.IMergeIterator; +import org.apache.cassandra.utils.MergeIterator; +import org.apache.cassandra.utils.memory.AbstractAllocator; + +/** + * Static methods to work with atom iterators. + */ +public abstract class UnfilteredRowIterators +{ + private static final Logger logger = LoggerFactory.getLogger(UnfilteredRowIterators.class); + + private UnfilteredRowIterators() {} + + public interface MergeListener + { + public void onMergePartitionLevelDeletion(DeletionTime mergedDeletion, DeletionTime[] versions); + + public void onMergingRows(Clustering clustering, LivenessInfo mergedInfo, DeletionTime mergedDeletion, Row[] versions); + public void onMergedComplexDeletion(ColumnDefinition c, DeletionTime mergedComplexDeletion, DeletionTime[] versions); + public void onMergedCells(Cell mergedCell, Cell[] versions); + public void onRowDone(); + + public void onMergedRangeTombstoneMarkers(RangeTombstoneMarker merged, RangeTombstoneMarker[] versions); + + public void close(); + } + + /** + * Returns a iterator that only returns rows with only live content. + * + * This is mainly used in the CQL layer when we know we don't care about deletion + * infos (and since an UnfilteredRowIterator cannot shadow it's own data, we know everyting + * returned isn't shadowed by a tombstone). + */ + public static RowIterator filter(UnfilteredRowIterator iter, int nowInSec) + { + return new FilteringIterator(iter, nowInSec); + + } + + /** + * Returns an iterator that is the result of merging other iterators. + */ + public static UnfilteredRowIterator merge(List iterators, int nowInSec) + { + assert !iterators.isEmpty(); + if (iterators.size() == 1) + return iterators.get(0); + + return UnfilteredRowMergeIterator.create(iterators, nowInSec, null); + } + + /** + * Returns an iterator that is the result of merging other iterators, and using + * specific MergeListener. + * + * Note that this method assumes that there is at least 2 iterators to merge. + */ + public static UnfilteredRowIterator merge(List iterators, int nowInSec, MergeListener mergeListener) + { + assert mergeListener != null; + return UnfilteredRowMergeIterator.create(iterators, nowInSec, mergeListener); + } + + /** + * Returns an empty atom iterator for a given partition. + */ + public static UnfilteredRowIterator emptyIterator(final CFMetaData cfm, final DecoratedKey partitionKey, final boolean isReverseOrder) + { + return new UnfilteredRowIterator() + { + public CFMetaData metadata() + { + return cfm; + } + + public boolean isReverseOrder() + { + return isReverseOrder; + } + + public PartitionColumns columns() + { + return PartitionColumns.NONE; + } + + public DecoratedKey partitionKey() + { + return partitionKey; + } + + public DeletionTime partitionLevelDeletion() + { + return DeletionTime.LIVE; + } + + public Row staticRow() + { + return Rows.EMPTY_STATIC_ROW; + } + + public RowStats stats() + { + return RowStats.NO_STATS; + } + + public boolean hasNext() + { + return false; + } + + public Unfiltered next() + { + throw new NoSuchElementException(); + } + + public void remove() + { + } + + public void close() + { + } + }; + } + + public static void digest(UnfilteredRowIterator iterator, MessageDigest digest) + { + // TODO: we're not computing digest the same way that old nodes. This + // means we'll have digest mismatches during upgrade. We should pass the messaging version of + // the node this is for (which might mean computing the digest last, and won't work + // for schema (where we announce the version through gossip to everyone)) + digest.update(iterator.partitionKey().getKey().duplicate()); + iterator.partitionLevelDeletion().digest(digest); + iterator.columns().digest(digest); + FBUtilities.updateWithBoolean(digest, iterator.isReverseOrder()); + iterator.staticRow().digest(digest); + + while (iterator.hasNext()) + { + Unfiltered unfiltered = iterator.next(); + if (unfiltered.kind() == Unfiltered.Kind.ROW) + ((Row) unfiltered).digest(digest); + else + ((RangeTombstoneMarker) unfiltered).digest(digest); + } + } + + /** + * Returns an iterator that concatenate two atom iterators. + * This method assumes that both iterator are from the same partition and that the atom from + * {@code iter2} come after the ones of {@code iter1} (that is, that concatenating the iterator + * make sense). + */ + public static UnfilteredRowIterator concat(final UnfilteredRowIterator iter1, final UnfilteredRowIterator iter2) + { + assert iter1.metadata().cfId.equals(iter2.metadata().cfId) + && iter1.partitionKey().equals(iter2.partitionKey()) + && iter1.partitionLevelDeletion().equals(iter2.partitionLevelDeletion()) + && iter1.isReverseOrder() == iter2.isReverseOrder() + && iter1.columns().equals(iter2.columns()) + && iter1.staticRow().equals(iter2.staticRow()); + + return new AbstractUnfilteredRowIterator(iter1.metadata(), + iter1.partitionKey(), + iter1.partitionLevelDeletion(), + iter1.columns(), + iter1.staticRow(), + iter1.isReverseOrder(), + iter1.stats()) + { + protected Unfiltered computeNext() + { + if (iter1.hasNext()) + return iter1.next(); + + return iter2.hasNext() ? iter2.next() : endOfData(); + } + + @Override + public void close() + { + try + { + iter1.close(); + } + finally + { + iter2.close(); + } + } + }; + } + + public static UnfilteredRowIterator cloningIterator(UnfilteredRowIterator iterator, final AbstractAllocator allocator) + { + return new WrappingUnfilteredRowIterator(iterator) + { + private final CloningRow cloningRow = new CloningRow(); + private final RangeTombstoneMarker.Builder markerBuilder = new RangeTombstoneMarker.Builder(iterator.metadata().comparator.size()); + + public Unfiltered next() + { + Unfiltered next = super.next(); + return next.kind() == Unfiltered.Kind.ROW + ? cloningRow.setTo((Row)next) + : clone((RangeTombstoneMarker)next); + } + + private RangeTombstoneMarker clone(RangeTombstoneMarker marker) + { + markerBuilder.reset(); + + RangeTombstone.Bound bound = marker.clustering(); + for (int i = 0; i < bound.size(); i++) + markerBuilder.writeClusteringValue(allocator.clone(bound.get(i))); + markerBuilder.writeBoundKind(bound.kind()); + if (marker.isBoundary()) + { + RangeTombstoneBoundaryMarker bm = (RangeTombstoneBoundaryMarker)marker; + markerBuilder.writeBoundaryDeletion(bm.endDeletionTime(), bm.startDeletionTime()); + } + else + { + markerBuilder.writeBoundDeletion(((RangeTombstoneBoundMarker)marker).deletionTime()); + } + markerBuilder.endOfMarker(); + return markerBuilder.build(); + } + + class CloningRow extends WrappingRow + { + private final CloningClustering cloningClustering = new CloningClustering(); + private final CloningCell cloningCell = new CloningCell(); + + protected Cell filterCell(Cell cell) + { + return cloningCell.setTo(cell); + } + + @Override + public Clustering clustering() + { + return cloningClustering.setTo(super.clustering()); + } + } + + class CloningClustering extends Clustering + { + private Clustering wrapped; + + public Clustering setTo(Clustering wrapped) + { + this.wrapped = wrapped; + return this; + } + + public int size() + { + return wrapped.size(); + } + + public ByteBuffer get(int i) + { + ByteBuffer value = wrapped.get(i); + return value == null ? null : allocator.clone(value); + } + + public ByteBuffer[] getRawValues() + { + throw new UnsupportedOperationException(); + } + } + + class CloningCell extends AbstractCell + { + private Cell wrapped; + + public Cell setTo(Cell wrapped) + { + this.wrapped = wrapped; + return this; + } + + public ColumnDefinition column() + { + return wrapped.column(); + } + + public boolean isCounterCell() + { + return wrapped.isCounterCell(); + } + + public ByteBuffer value() + { + return allocator.clone(wrapped.value()); + } + + public LivenessInfo livenessInfo() + { + return wrapped.livenessInfo(); + } + + public CellPath path() + { + CellPath path = wrapped.path(); + if (path == null) + return null; + + assert path.size() == 1; + return CellPath.create(allocator.clone(path.get(0))); + } + } + }; + } + + /** + * Turns the given iterator into an update. + * + * Warning: this method does not close the provided iterator, it is up to + * the caller to close it. + */ + public static PartitionUpdate toUpdate(UnfilteredRowIterator iterator) + { + PartitionUpdate update = new PartitionUpdate(iterator.metadata(), iterator.partitionKey(), iterator.columns(), 1); + + update.addPartitionDeletion(iterator.partitionLevelDeletion()); + + if (iterator.staticRow() != Rows.EMPTY_STATIC_ROW) + iterator.staticRow().copyTo(update.staticWriter()); + + while (iterator.hasNext()) + { + Unfiltered unfiltered = iterator.next(); + if (unfiltered.kind() == Unfiltered.Kind.ROW) + ((Row) unfiltered).copyTo(update.writer()); + else + ((RangeTombstoneMarker) unfiltered).copyTo(update.markerWriter(iterator.isReverseOrder())); + } + + return update; + } + + /** + * Validate that the data of the provided iterator is valid, that is that the values + * it contains are valid for the type they represent, and more generally that the + * infos stored are sensible. + * + * This is mainly used by scrubber to detect problems in sstables. + * + * @param iterator the partition to check. + * @param filename the name of the file the data is comming from. + * @return an iterator that returns the same data than {@code iterator} but that + * checks said data and throws a {@code CorruptedSSTableException} if it detects + * invalid data. + */ + public static UnfilteredRowIterator withValidation(UnfilteredRowIterator iterator, final String filename) + { + return new WrappingUnfilteredRowIterator(iterator) + { + public Unfiltered next() + { + Unfiltered next = super.next(); + try + { + next.validateData(metadata()); + return next; + } + catch (MarshalException me) + { + throw new CorruptSSTableException(me, filename); + } + } + }; + } + + /** + * Convert all expired cells to equivalent tombstones. + *

+ * Once a cell expires, it acts exactly as a tombstone and this until it is purged. But in particular that + * means we don't care about the value of an expired cell, and it is thus equivalent but more efficient to + * replace the expired cell by an equivalent tombstone (that has no value). + * + * @param iterator the iterator in which to conver expired cells. + * @param nowInSec the current time to use to decide if a cell is expired. + * @return an iterator that returns the same data than {@code iterator} but with all expired cells converted + * to equivalent tombstones. + */ + public static UnfilteredRowIterator convertExpiredCellsToTombstones(UnfilteredRowIterator iterator, final int nowInSec) + { + return new FilteringRowIterator(iterator) + { + protected FilteringRow makeRowFilter() + { + return new FilteringRow() + { + @Override + protected Cell filterCell(Cell cell) + { + Cell filtered = super.filterCell(cell); + if (filtered == null) + return null; + + LivenessInfo info = filtered.livenessInfo(); + if (info.hasTTL() && !filtered.isLive(nowInSec)) + { + // The column is now expired, we can safely return a simple tombstone. Note that as long as the expiring + // column and the tombstone put together live longer than GC grace seconds, we'll fulfil our responsibility + // to repair. See discussion at + // http://cassandra-user-incubator-apache-org.3065146.n2.nabble.com/repair-compaction-and-tombstone-rows-td7583481.html + return Cells.create(filtered.column(), + filtered.isCounterCell(), + ByteBufferUtil.EMPTY_BYTE_BUFFER, + SimpleLivenessInfo.forDeletion(info.timestamp(), info.localDeletionTime() - info.ttl()), + filtered.path()); + } + else + { + return filtered; + } + } + }; + } + }; + } + + /** + * Wraps the provided iterator so it logs the returned atoms for debugging purposes. + *

+ * Note that this is only meant for debugging as this can log a very large amount of + * logging at INFO. + */ + public static UnfilteredRowIterator loggingIterator(UnfilteredRowIterator iterator, final String id, final boolean fullDetails) + { + CFMetaData metadata = iterator.metadata(); + logger.info("[{}] Logging iterator on {}.{}, partition key={}, reversed={}, deletion={}", + id, + metadata.ksName, + metadata.cfName, + metadata.getKeyValidator().getString(iterator.partitionKey().getKey()), + iterator.isReverseOrder(), + iterator.partitionLevelDeletion().markedForDeleteAt()); + + return new WrappingUnfilteredRowIterator(iterator) + { + @Override + public Row staticRow() + { + Row row = super.staticRow(); + if (!row.isEmpty()) + logger.info("[{}] {}", id, row.toString(metadata(), fullDetails)); + return row; + } + + @Override + public Unfiltered next() + { + Unfiltered next = super.next(); + if (next.kind() == Unfiltered.Kind.ROW) + logger.info("[{}] {}", id, ((Row)next).toString(metadata(), fullDetails)); + else + logger.info("[{}] {}", id, ((RangeTombstoneMarker)next).toString(metadata())); + return next; + } + }; + } + + /** + * A wrapper over MergeIterator to implement the UnfilteredRowIterator interface. + */ + private static class UnfilteredRowMergeIterator extends AbstractUnfilteredRowIterator + { + private final IMergeIterator mergeIterator; + private final MergeListener listener; + + private UnfilteredRowMergeIterator(CFMetaData metadata, + List iterators, + PartitionColumns columns, + DeletionTime partitionDeletion, + int nowInSec, + boolean reversed, + MergeListener listener) + { + super(metadata, + iterators.get(0).partitionKey(), + partitionDeletion, + columns, + mergeStaticRows(metadata, iterators, columns.statics, nowInSec, listener, partitionDeletion), + reversed, + mergeStats(iterators)); + + this.listener = listener; + this.mergeIterator = MergeIterator.get(iterators, + reversed ? metadata.comparator.reversed() : metadata.comparator, + new MergeReducer(metadata, iterators.size(), reversed, nowInSec)); + } + + private static UnfilteredRowMergeIterator create(List iterators, int nowInSec, MergeListener listener) + { + try + { + checkForInvalidInput(iterators); + return new UnfilteredRowMergeIterator(iterators.get(0).metadata(), + iterators, + collectColumns(iterators), + collectPartitionLevelDeletion(iterators, listener), + nowInSec, + iterators.get(0).isReverseOrder(), + listener); + } + catch (RuntimeException | Error e) + { + try + { + FBUtilities.closeAll(iterators); + } + catch (Exception suppressed) + { + e.addSuppressed(suppressed); + } + throw e; + } + } + + @SuppressWarnings("resource") // We're not really creating any resource here + private static void checkForInvalidInput(List iterators) + { + if (iterators.isEmpty()) + return; + + UnfilteredRowIterator first = iterators.get(0); + for (int i = 1; i < iterators.size(); i++) + { + UnfilteredRowIterator iter = iterators.get(i); + assert first.metadata().cfId.equals(iter.metadata().cfId); + assert first.partitionKey().equals(iter.partitionKey()); + assert first.isReverseOrder() == iter.isReverseOrder(); + } + } + + @SuppressWarnings("resource") // We're not really creating any resource here + private static DeletionTime collectPartitionLevelDeletion(List iterators, MergeListener listener) + { + DeletionTime[] versions = listener == null ? null : new DeletionTime[iterators.size()]; + + DeletionTime delTime = DeletionTime.LIVE; + for (int i = 0; i < iterators.size(); i++) + { + UnfilteredRowIterator iter = iterators.get(i); + DeletionTime iterDeletion = iter.partitionLevelDeletion(); + if (listener != null) + versions[i] = iterDeletion; + if (!delTime.supersedes(iterDeletion)) + delTime = iterDeletion; + } + if (listener != null && !delTime.isLive()) + listener.onMergePartitionLevelDeletion(delTime, versions); + return delTime; + } + + private static Row mergeStaticRows(CFMetaData metadata, + List iterators, + Columns columns, + int nowInSec, + MergeListener listener, + DeletionTime partitionDeletion) + { + if (columns.isEmpty()) + return Rows.EMPTY_STATIC_ROW; + + Row.Merger merger = Row.Merger.createStatic(metadata, iterators.size(), nowInSec, columns, listener); + for (int i = 0; i < iterators.size(); i++) + merger.add(i, iterators.get(i).staticRow()); + + // Note that we should call 'takeAlias' on the result in theory, but we know that we + // won't reuse the merger and so that it's ok not to. + Row merged = merger.merge(partitionDeletion); + return merged == null ? Rows.EMPTY_STATIC_ROW : merged; + } + + private static PartitionColumns collectColumns(List iterators) + { + PartitionColumns first = iterators.get(0).columns(); + Columns statics = first.statics; + Columns regulars = first.regulars; + for (int i = 1; i < iterators.size(); i++) + { + PartitionColumns cols = iterators.get(i).columns(); + statics = statics.mergeTo(cols.statics); + regulars = regulars.mergeTo(cols.regulars); + } + return statics == first.statics && regulars == first.regulars + ? first + : new PartitionColumns(statics, regulars); + } + + private static RowStats mergeStats(List iterators) + { + RowStats stats = RowStats.NO_STATS; + for (UnfilteredRowIterator iter : iterators) + stats = stats.mergeWith(iter.stats()); + return stats; + } + + protected Unfiltered computeNext() + { + while (mergeIterator.hasNext()) + { + Unfiltered merged = mergeIterator.next(); + if (merged != null) + return merged; + } + return endOfData(); + } + + public void close() + { + // This will close the input iterators + FileUtils.closeQuietly(mergeIterator); + + if (listener != null) + listener.close(); + } + + /** + * Specific reducer for merge operations that rewrite the same reusable + * row every time. This also skip cells shadowed by range tombstones when writing. + */ + private class MergeReducer extends MergeIterator.Reducer + { + private Unfiltered.Kind nextKind; + + private final Row.Merger rowMerger; + private final RangeTombstoneMarker.Merger markerMerger; + + private MergeReducer(CFMetaData metadata, int size, boolean reversed, int nowInSec) + { + this.rowMerger = Row.Merger.createRegular(metadata, size, nowInSec, columns().regulars, listener); + this.markerMerger = new RangeTombstoneMarker.Merger(metadata, size, partitionLevelDeletion(), reversed, listener); + } + + @Override + public boolean trivialReduceIsTrivial() + { + return listener == null; + } + + public void reduce(int idx, Unfiltered current) + { + nextKind = current.kind(); + if (nextKind == Unfiltered.Kind.ROW) + rowMerger.add(idx, (Row)current); + else + markerMerger.add(idx, (RangeTombstoneMarker)current); + } + + protected Unfiltered getReduced() + { + return nextKind == Unfiltered.Kind.ROW + ? rowMerger.merge(markerMerger.activeDeletion()) + : markerMerger.merge(); + } + + protected void onKeyChange() + { + if (nextKind == Unfiltered.Kind.ROW) + rowMerger.clear(); + else + markerMerger.clear(); + } + } + } + + private static class FilteringIterator extends AbstractIterator implements RowIterator + { + private final UnfilteredRowIterator iter; + private final int nowInSec; + private final TombstoneFilteringRow filter; + + public FilteringIterator(UnfilteredRowIterator iter, int nowInSec) + { + this.iter = iter; + this.nowInSec = nowInSec; + this.filter = new TombstoneFilteringRow(nowInSec); + } + + public CFMetaData metadata() + { + return iter.metadata(); + } + + public boolean isReverseOrder() + { + return iter.isReverseOrder(); + } + + public PartitionColumns columns() + { + return iter.columns(); + } + + public DecoratedKey partitionKey() + { + return iter.partitionKey(); + } + + public Row staticRow() + { + Row row = iter.staticRow(); + return row.isEmpty() ? row : new TombstoneFilteringRow(nowInSec).setTo(row); + } + + protected Row computeNext() + { + while (iter.hasNext()) + { + Unfiltered next = iter.next(); + if (next.kind() != Unfiltered.Kind.ROW) + continue; + + Row row = filter.setTo((Row)next); + if (!row.isEmpty()) + return row; + } + return endOfData(); + } + + public void close() + { + iter.close(); + } + } +} diff --git a/src/java/org/apache/cassandra/db/rows/UnfilteredSerializer.java b/src/java/org/apache/cassandra/db/rows/UnfilteredSerializer.java new file mode 100644 index 0000000000..a5a0c756de --- /dev/null +++ b/src/java/org/apache/cassandra/db/rows/UnfilteredSerializer.java @@ -0,0 +1,706 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.cassandra.db.rows; + +import java.io.DataInput; +import java.io.IOException; +import java.nio.ByteBuffer; +import java.util.*; + +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import org.apache.cassandra.config.ColumnDefinition; +import org.apache.cassandra.db.*; +import org.apache.cassandra.io.util.DataOutputPlus; +import org.apache.cassandra.utils.ByteBufferUtil; +import org.apache.cassandra.utils.SearchIterator; + +/** + * Serialize/deserialize a single Unfiltered for the intra-node protocol. + * + * The encode format for an unfiltered is (|) where: + * + * is a byte whose bits are flags. The rightmost 1st bit is only + * set to indicate the end of the partition. The 2nd bit indicates + * whether the reminder is a range tombstone marker (otherwise it's a row). + * If it's a row then the 3rd bit indicates if it's static, the 4th bit + * indicates the presence of a row timestamp, the 5th the presence of a row + * ttl, the 6th the presence of row deletion and the 7th indicates the + * presence of complex deletion times. + * is [][][]...... where + * is the row clustering as serialized by + * {@code Clustering.serializer}. Note that static row are an exception and + * don't have this. , and are the row timestamp, ttl and deletion + * whose presence is determined by the flags. is the simple columns of the row and the + * complex ones. There is actually 2 slightly different possible layout for those + * cell: a dense one and a sparse one. Which one is used depends on the serialization + * header and more precisely of {@link SerializationHeader.useSparseColumnLayout()}: + * 1) in the dense layout, there will be as many and as there is columns + * in the serialization header. *Each simple column will simply be a + * (which might have no value, see below), while each will be + * []... where is the deletion for + * this complex column (if flags indicates it present), are the + * for this complex column and is a last cell that will have no value + * to indicate the end of this column. + * 2) in the sparse layout, there won't be "empty" cells, i.e. only the column that + * actually have a cell are represented. For that, each and start + * by a 2 byte index that points to the column in the header it belongs to. After + * that, each and is the same than for the dense layout. But contrarily + * to the dense layout we won't know how many elements are serialized so a 2 byte + * marker with a value of -1 will indicates the end of the row. + * is where is the marker bound as serialized + * by {@code Slice.Bound.serializer} and is the marker deletion + * time. + * + * A cell start with a 1 byte . Thre rightmost 1st bit indicates + * if there is actually a value for this cell. If this flag is unset, + * nothing more follows for the cell. The 2nd and third flag indicates if + * it's a deleted or expiring cell. The 4th flag indicates if the value + * is empty or not. The 5th and 6th indicates if the timestamp and ttl/ + * localDeletionTime for the cell are the same than the row one (if that + * is the case, those are not repeated for the cell).Follows the + * (unless it's marked empty in the flag) and a delta-encoded long + * (unless the flag tells to use the row level one). + * Then if it's a deleted or expiring cell a delta-encoded int + * and if it's expiring a delta-encoded int (unless it's an expiring cell + * and the ttl and localDeletionTime are indicated by the flags to be the same + * than the row ones, in which case none of those appears). + */ +public class UnfilteredSerializer +{ + private static final Logger logger = LoggerFactory.getLogger(UnfilteredSerializer.class); + + public static final UnfilteredSerializer serializer = new UnfilteredSerializer(); + + // Unfiltered flags + private final static int END_OF_PARTITION = 0x01; + private final static int IS_MARKER = 0x02; + // For rows + private final static int IS_STATIC = 0x04; + private final static int HAS_TIMESTAMP = 0x08; + private final static int HAS_TTL = 0x10; + private final static int HAS_DELETION = 0x20; + private final static int HAS_COMPLEX_DELETION = 0x40; + + // Cell flags + private final static int PRESENCE_MASK = 0x01; + private final static int DELETION_MASK = 0x02; + private final static int EXPIRATION_MASK = 0x04; + private final static int EMPTY_VALUE_MASK = 0x08; + private final static int USE_ROW_TIMESTAMP = 0x10; + private final static int USE_ROW_TTL = 0x20; + + public void serialize(Unfiltered unfiltered, SerializationHeader header, DataOutputPlus out, int version) + throws IOException + { + if (unfiltered.kind() == Unfiltered.Kind.RANGE_TOMBSTONE_MARKER) + { + serialize((RangeTombstoneMarker) unfiltered, header, out, version); + } + else + { + serialize((Row) unfiltered, header, out, version); + } + } + + public void serialize(Row row, SerializationHeader header, DataOutputPlus out, int version) + throws IOException + { + int flags = 0; + boolean isStatic = row.isStatic(); + + LivenessInfo pkLiveness = row.primaryKeyLivenessInfo(); + DeletionTime deletion = row.deletion(); + boolean hasComplexDeletion = row.hasComplexDeletion(); + + if (isStatic) + flags |= IS_STATIC; + if (pkLiveness.hasTimestamp()) + flags |= HAS_TIMESTAMP; + if (pkLiveness.hasTTL()) + flags |= HAS_TTL; + if (!deletion.isLive()) + flags |= HAS_DELETION; + if (hasComplexDeletion) + flags |= HAS_COMPLEX_DELETION; + + out.writeByte((byte)flags); + if (!isStatic) + Clustering.serializer.serialize(row.clustering(), out, version, header.clusteringTypes()); + + if ((flags & HAS_TIMESTAMP) != 0) + out.writeLong(header.encodeTimestamp(pkLiveness.timestamp())); + if ((flags & HAS_TTL) != 0) + { + out.writeInt(header.encodeTTL(pkLiveness.ttl())); + out.writeInt(header.encodeDeletionTime(pkLiveness.localDeletionTime())); + } + if ((flags & HAS_DELETION) != 0) + UnfilteredRowIteratorSerializer.writeDelTime(deletion, header, out); + + Columns columns = header.columns(isStatic); + int simpleCount = columns.simpleColumnCount(); + boolean useSparse = header.useSparseColumnLayout(isStatic); + SearchIterator cells = row.searchIterator(); + + for (int i = 0; i < simpleCount; i++) + writeSimpleColumn(i, cells.next(columns.getSimple(i)), header, out, pkLiveness, useSparse); + + for (int i = simpleCount; i < columns.columnCount(); i++) + writeComplexColumn(i, cells.next(columns.getComplex(i - simpleCount)), hasComplexDeletion, header, out, pkLiveness, useSparse); + + if (useSparse) + out.writeShort(-1); + } + + private void writeSimpleColumn(int idx, ColumnData data, SerializationHeader header, DataOutputPlus out, LivenessInfo rowLiveness, boolean useSparse) + throws IOException + { + if (useSparse) + { + if (data == null) + return; + + out.writeShort(idx); + } + + writeCell(data == null ? null : data.cell(), header, out, rowLiveness); + } + + private void writeComplexColumn(int idx, ColumnData data, boolean hasComplexDeletion, SerializationHeader header, DataOutputPlus out, LivenessInfo rowLiveness, boolean useSparse) + throws IOException + { + Iterator cells = data == null ? null : data.cells(); + DeletionTime deletion = data == null ? DeletionTime.LIVE : data.complexDeletion(); + + if (useSparse) + { + assert hasComplexDeletion || deletion.isLive(); + if (cells == null && deletion.isLive()) + return; + + out.writeShort(idx); + } + + if (hasComplexDeletion) + UnfilteredRowIteratorSerializer.writeDelTime(deletion, header, out); + + if (cells != null) + while (cells.hasNext()) + writeCell(cells.next(), header, out, rowLiveness); + + writeCell(null, header, out, rowLiveness); + } + + public void serialize(RangeTombstoneMarker marker, SerializationHeader header, DataOutputPlus out, int version) + throws IOException + { + out.writeByte((byte)IS_MARKER); + RangeTombstone.Bound.serializer.serialize(marker.clustering(), out, version, header.clusteringTypes()); + + if (marker.isBoundary()) + { + RangeTombstoneBoundaryMarker bm = (RangeTombstoneBoundaryMarker)marker; + UnfilteredRowIteratorSerializer.writeDelTime(bm.endDeletionTime(), header, out); + UnfilteredRowIteratorSerializer.writeDelTime(bm.startDeletionTime(), header, out); + } + else + { + UnfilteredRowIteratorSerializer.writeDelTime(((RangeTombstoneBoundMarker)marker).deletionTime(), header, out); + } + } + + public long serializedSize(Unfiltered unfiltered, SerializationHeader header, int version, TypeSizes sizes) + { + return unfiltered.kind() == Unfiltered.Kind.RANGE_TOMBSTONE_MARKER + ? serializedSize((RangeTombstoneMarker) unfiltered, header, version, sizes) + : serializedSize((Row) unfiltered, header, version, sizes); + } + + public long serializedSize(Row row, SerializationHeader header, int version, TypeSizes sizes) + { + long size = 1; // flags + + boolean isStatic = row.isStatic(); + LivenessInfo pkLiveness = row.primaryKeyLivenessInfo(); + DeletionTime deletion = row.deletion(); + boolean hasComplexDeletion = row.hasComplexDeletion(); + + if (!isStatic) + size += Clustering.serializer.serializedSize(row.clustering(), version, header.clusteringTypes(), sizes); + + if (pkLiveness.hasTimestamp()) + size += sizes.sizeof(header.encodeTimestamp(pkLiveness.timestamp())); + if (pkLiveness.hasTTL()) + { + size += sizes.sizeof(header.encodeTTL(pkLiveness.ttl())); + size += sizes.sizeof(header.encodeDeletionTime(pkLiveness.localDeletionTime())); + } + if (!deletion.isLive()) + size += UnfilteredRowIteratorSerializer.delTimeSerializedSize(deletion, header, sizes); + + Columns columns = header.columns(isStatic); + int simpleCount = columns.simpleColumnCount(); + boolean useSparse = header.useSparseColumnLayout(isStatic); + SearchIterator cells = row.searchIterator(); + + for (int i = 0; i < simpleCount; i++) + size += sizeOfSimpleColumn(i, cells.next(columns.getSimple(i)), header, sizes, pkLiveness, useSparse); + + for (int i = simpleCount; i < columns.columnCount(); i++) + size += sizeOfComplexColumn(i, cells.next(columns.getComplex(i - simpleCount)), hasComplexDeletion, header, sizes, pkLiveness, useSparse); + + if (useSparse) + size += sizes.sizeof((short)-1); + + return size; + } + + private long sizeOfSimpleColumn(int idx, ColumnData data, SerializationHeader header, TypeSizes sizes, LivenessInfo rowLiveness, boolean useSparse) + { + long size = 0; + if (useSparse) + { + if (data == null) + return size; + + size += sizes.sizeof((short)idx); + } + return size + sizeOfCell(data == null ? null : data.cell(), header, sizes, rowLiveness); + } + + private long sizeOfComplexColumn(int idx, ColumnData data, boolean hasComplexDeletion, SerializationHeader header, TypeSizes sizes, LivenessInfo rowLiveness, boolean useSparse) + { + long size = 0; + Iterator cells = data == null ? null : data.cells(); + DeletionTime deletion = data == null ? DeletionTime.LIVE : data.complexDeletion(); + if (useSparse) + { + assert hasComplexDeletion || deletion.isLive(); + if (cells == null && deletion.isLive()) + return size; + + size += sizes.sizeof((short)idx); + } + + if (hasComplexDeletion) + size += UnfilteredRowIteratorSerializer.delTimeSerializedSize(deletion, header, sizes); + + if (cells != null) + while (cells.hasNext()) + size += sizeOfCell(cells.next(), header, sizes, rowLiveness); + + return size + sizeOfCell(null, header, sizes, rowLiveness); + } + + public long serializedSize(RangeTombstoneMarker marker, SerializationHeader header, int version, TypeSizes sizes) + { + long size = 1 // flags + + RangeTombstone.Bound.serializer.serializedSize(marker.clustering(), version, header.clusteringTypes(), sizes); + + if (marker.isBoundary()) + { + RangeTombstoneBoundaryMarker bm = (RangeTombstoneBoundaryMarker)marker; + size += UnfilteredRowIteratorSerializer.delTimeSerializedSize(bm.endDeletionTime(), header, sizes); + size += UnfilteredRowIteratorSerializer.delTimeSerializedSize(bm.startDeletionTime(), header, sizes); + } + else + { + size += UnfilteredRowIteratorSerializer.delTimeSerializedSize(((RangeTombstoneBoundMarker)marker).deletionTime(), header, sizes); + } + return size; + } + + public void writeEndOfPartition(DataOutputPlus out) throws IOException + { + out.writeByte((byte)1); + } + + public long serializedSizeEndOfPartition(TypeSizes sizes) + { + return 1; + } + + public Unfiltered.Kind deserialize(DataInput in, + SerializationHeader header, + SerializationHelper helper, + Row.Writer rowWriter, + RangeTombstoneMarker.Writer markerWriter) + throws IOException + { + int flags = in.readUnsignedByte(); + if (isEndOfPartition(flags)) + return null; + + if (kind(flags) == Unfiltered.Kind.RANGE_TOMBSTONE_MARKER) + { + RangeTombstone.Bound.Kind kind = RangeTombstone.Bound.serializer.deserialize(in, helper.version, header.clusteringTypes(), markerWriter); + deserializeMarkerBody(in, header, kind.isBoundary(), markerWriter); + return Unfiltered.Kind.RANGE_TOMBSTONE_MARKER; + } + else + { + assert !isStatic(flags); // deserializeStaticRow should be used for that. + Clustering.serializer.deserialize(in, helper.version, header.clusteringTypes(), rowWriter); + deserializeRowBody(in, header, helper, flags, rowWriter); + return Unfiltered.Kind.ROW; + } + } + + public Row deserializeStaticRow(DataInput in, SerializationHeader header, SerializationHelper helper) + throws IOException + { + int flags = in.readUnsignedByte(); + assert !isEndOfPartition(flags) && kind(flags) == Unfiltered.Kind.ROW && isStatic(flags); + StaticRow.Builder builder = StaticRow.builder(header.columns().statics, true, header.columns().statics.hasCounters()); + deserializeRowBody(in, header, helper, flags, builder); + return builder.build(); + } + + public void skipStaticRow(DataInput in, SerializationHeader header, SerializationHelper helper) throws IOException + { + int flags = in.readUnsignedByte(); + assert !isEndOfPartition(flags) && kind(flags) == Unfiltered.Kind.ROW && isStatic(flags) : "Flags is " + flags; + skipRowBody(in, header, helper, flags); + } + + public void deserializeMarkerBody(DataInput in, + SerializationHeader header, + boolean isBoundary, + RangeTombstoneMarker.Writer writer) + throws IOException + { + if (isBoundary) + writer.writeBoundaryDeletion(UnfilteredRowIteratorSerializer.readDelTime(in, header), UnfilteredRowIteratorSerializer.readDelTime(in, header)); + else + writer.writeBoundDeletion(UnfilteredRowIteratorSerializer.readDelTime(in, header)); + writer.endOfMarker(); + } + + public void skipMarkerBody(DataInput in, SerializationHeader header, boolean isBoundary) throws IOException + { + if (isBoundary) + { + UnfilteredRowIteratorSerializer.skipDelTime(in, header); + UnfilteredRowIteratorSerializer.skipDelTime(in, header); + } + else + { + UnfilteredRowIteratorSerializer.skipDelTime(in, header); + } + } + + public void deserializeRowBody(DataInput in, + SerializationHeader header, + SerializationHelper helper, + int flags, + Row.Writer writer) + throws IOException + { + boolean isStatic = isStatic(flags); + boolean hasTimestamp = (flags & HAS_TIMESTAMP) != 0; + boolean hasTTL = (flags & HAS_TTL) != 0; + boolean hasDeletion = (flags & HAS_DELETION) != 0; + boolean hasComplexDeletion = (flags & HAS_COMPLEX_DELETION) != 0; + + long timestamp = hasTimestamp ? header.decodeTimestamp(in.readLong()) : LivenessInfo.NO_TIMESTAMP; + int ttl = hasTTL ? header.decodeTTL(in.readInt()) : LivenessInfo.NO_TTL; + int localDeletionTime = hasTTL ? header.decodeDeletionTime(in.readInt()) : LivenessInfo.NO_DELETION_TIME; + DeletionTime deletion = hasDeletion ? UnfilteredRowIteratorSerializer.readDelTime(in, header) : DeletionTime.LIVE; + + helper.writePartitionKeyLivenessInfo(writer, timestamp, ttl, localDeletionTime); + writer.writeRowDeletion(deletion); + + Columns columns = header.columns(isStatic); + if (header.useSparseColumnLayout(isStatic)) + { + int count = columns.columnCount(); + int simpleCount = columns.simpleColumnCount(); + int i; + while ((i = in.readShort()) >= 0) + { + if (i > count) + throw new IOException(String.format("Impossible column index %d, the header has only %d columns defined", i, count)); + + if (i < simpleCount) + readSimpleColumn(columns.getSimple(i), in, header, helper, writer); + else + readComplexColumn(columns.getComplex(i - simpleCount), in, header, helper, hasComplexDeletion, writer); + } + } + else + { + for (int i = 0; i < columns.simpleColumnCount(); i++) + readSimpleColumn(columns.getSimple(i), in, header, helper, writer); + + for (int i = 0; i < columns.complexColumnCount(); i++) + readComplexColumn(columns.getComplex(i), in, header, helper, hasComplexDeletion, writer); + } + + writer.endOfRow(); + } + + private void readSimpleColumn(ColumnDefinition column, DataInput in, SerializationHeader header, SerializationHelper helper, Row.Writer writer) + throws IOException + { + if (helper.includes(column)) + readCell(column, in, header, helper, writer); + else + skipCell(column, in, header); + } + + private void readComplexColumn(ColumnDefinition column, DataInput in, SerializationHeader header, SerializationHelper helper, boolean hasComplexDeletion, Row.Writer writer) + throws IOException + { + if (helper.includes(column)) + { + helper.startOfComplexColumn(column); + + if (hasComplexDeletion) + writer.writeComplexDeletion(column, UnfilteredRowIteratorSerializer.readDelTime(in, header)); + + while (readCell(column, in, header, helper, writer)); + + helper.endOfComplexColumn(column); + } + else + { + skipComplexColumn(column, in, header, helper, hasComplexDeletion); + } + } + + public void skipRowBody(DataInput in, SerializationHeader header, SerializationHelper helper, int flags) throws IOException + { + boolean isStatic = isStatic(flags); + boolean hasTimestamp = (flags & HAS_TIMESTAMP) != 0; + boolean hasTTL = (flags & HAS_TTL) != 0; + boolean hasDeletion = (flags & HAS_DELETION) != 0; + boolean hasComplexDeletion = (flags & HAS_COMPLEX_DELETION) != 0; + + // Note that we don't want want to use FileUtils.skipBytesFully for anything that may not have + // the size we think due to VINT encoding + if (hasTimestamp) + in.readLong(); + if (hasTTL) + { + // ttl and localDeletionTime + in.readInt(); + in.readInt(); + } + if (hasDeletion) + UnfilteredRowIteratorSerializer.skipDelTime(in, header); + + Columns columns = header.columns(isStatic); + if (header.useSparseColumnLayout(isStatic)) + { + int count = columns.columnCount(); + int simpleCount = columns.simpleColumnCount(); + int i; + while ((i = in.readShort()) >= 0) + { + if (i > count) + throw new IOException(String.format("Impossible column index %d, the header has only %d columns defined", i, count)); + + if (i < simpleCount) + skipCell(columns.getSimple(i), in, header); + else + skipComplexColumn(columns.getComplex(i - simpleCount), in, header, helper, hasComplexDeletion); + } + } + else + { + for (int i = 0; i < columns.simpleColumnCount(); i++) + skipCell(columns.getSimple(i), in, header); + + for (int i = 0; i < columns.complexColumnCount(); i++) + skipComplexColumn(columns.getComplex(i), in, header, helper, hasComplexDeletion); + } + } + + private void skipComplexColumn(ColumnDefinition column, DataInput in, SerializationHeader header, SerializationHelper helper, boolean hasComplexDeletion) + throws IOException + { + if (hasComplexDeletion) + UnfilteredRowIteratorSerializer.skipDelTime(in, header); + + while (skipCell(column, in, header)); + } + + public static boolean isEndOfPartition(int flags) + { + return (flags & END_OF_PARTITION) != 0; + } + + public static Unfiltered.Kind kind(int flags) + { + return (flags & IS_MARKER) != 0 ? Unfiltered.Kind.RANGE_TOMBSTONE_MARKER : Unfiltered.Kind.ROW; + } + + public static boolean isStatic(int flags) + { + return (flags & IS_MARKER) == 0 && (flags & IS_STATIC) != 0; + } + + private void writeCell(Cell cell, SerializationHeader header, DataOutputPlus out, LivenessInfo rowLiveness) + throws IOException + { + if (cell == null) + { + out.writeByte((byte)0); + return; + } + + boolean hasValue = cell.value().hasRemaining(); + boolean isDeleted = cell.isTombstone(); + boolean isExpiring = cell.isExpiring(); + boolean useRowTimestamp = rowLiveness.hasTimestamp() && cell.livenessInfo().timestamp() == rowLiveness.timestamp(); + boolean useRowTTL = isExpiring && rowLiveness.hasTTL() && cell.livenessInfo().ttl() == rowLiveness.ttl() && cell.livenessInfo().localDeletionTime() == rowLiveness.localDeletionTime(); + int flags = PRESENCE_MASK; + if (!hasValue) + flags |= EMPTY_VALUE_MASK; + + if (isDeleted) + flags |= DELETION_MASK; + else if (isExpiring) + flags |= EXPIRATION_MASK; + + if (useRowTimestamp) + flags |= USE_ROW_TIMESTAMP; + if (useRowTTL) + flags |= USE_ROW_TTL; + + out.writeByte((byte)flags); + + if (hasValue) + header.getType(cell.column()).writeValue(cell.value(), out); + + if (!useRowTimestamp) + out.writeLong(header.encodeTimestamp(cell.livenessInfo().timestamp())); + + if ((isDeleted || isExpiring) && !useRowTTL) + out.writeInt(header.encodeDeletionTime(cell.livenessInfo().localDeletionTime())); + if (isExpiring && !useRowTTL) + out.writeInt(header.encodeTTL(cell.livenessInfo().ttl())); + + if (cell.column().isComplex()) + cell.column().cellPathSerializer().serialize(cell.path(), out); + } + + private long sizeOfCell(Cell cell, SerializationHeader header, TypeSizes sizes, LivenessInfo rowLiveness) + { + long size = 1; // flags + + if (cell == null) + return size; + + boolean hasValue = cell.value().hasRemaining(); + boolean isDeleted = cell.isTombstone(); + boolean isExpiring = cell.isExpiring(); + boolean useRowTimestamp = rowLiveness.hasTimestamp() && cell.livenessInfo().timestamp() == rowLiveness.timestamp(); + boolean useRowTTL = isExpiring && rowLiveness.hasTTL() && cell.livenessInfo().ttl() == rowLiveness.ttl() && cell.livenessInfo().localDeletionTime() == rowLiveness.localDeletionTime(); + + if (hasValue) + size += header.getType(cell.column()).writtenLength(cell.value(), sizes); + + if (!useRowTimestamp) + size += sizes.sizeof(header.encodeTimestamp(cell.livenessInfo().timestamp())); + + if ((isDeleted || isExpiring) && !useRowTTL) + size += sizes.sizeof(header.encodeDeletionTime(cell.livenessInfo().localDeletionTime())); + if (isExpiring && !useRowTTL) + size += sizes.sizeof(header.encodeTTL(cell.livenessInfo().ttl())); + + if (cell.column().isComplex()) + size += cell.column().cellPathSerializer().serializedSize(cell.path(), sizes); + + return size; + } + + private boolean readCell(ColumnDefinition column, DataInput in, SerializationHeader header, SerializationHelper helper, Row.Writer writer) + throws IOException + { + int flags = in.readUnsignedByte(); + if ((flags & PRESENCE_MASK) == 0) + return false; + + boolean hasValue = (flags & EMPTY_VALUE_MASK) == 0; + boolean isDeleted = (flags & DELETION_MASK) != 0; + boolean isExpiring = (flags & EXPIRATION_MASK) != 0; + boolean useRowTimestamp = (flags & USE_ROW_TIMESTAMP) != 0; + boolean useRowTTL = (flags & USE_ROW_TTL) != 0; + + ByteBuffer value = ByteBufferUtil.EMPTY_BYTE_BUFFER; + if (hasValue) + { + if (helper.canSkipValue(column)) + header.getType(column).skipValue(in); + else + value = header.getType(column).readValue(in); + } + + long timestamp = useRowTimestamp ? helper.getRowTimestamp() : header.decodeTimestamp(in.readLong()); + + int localDelTime = useRowTTL + ? helper.getRowLocalDeletionTime() + : (isDeleted || isExpiring ? header.decodeDeletionTime(in.readInt()) : LivenessInfo.NO_DELETION_TIME); + + int ttl = useRowTTL + ? helper.getRowTTL() + : (isExpiring ? header.decodeTTL(in.readInt()) : LivenessInfo.NO_TTL); + + CellPath path = column.isComplex() + ? column.cellPathSerializer().deserialize(in) + : null; + + helper.writeCell(writer, column, false, value, timestamp, localDelTime, ttl, path); + + return true; + } + + private boolean skipCell(ColumnDefinition column, DataInput in, SerializationHeader header) + throws IOException + { + int flags = in.readUnsignedByte(); + if ((flags & PRESENCE_MASK) == 0) + return false; + + boolean hasValue = (flags & EMPTY_VALUE_MASK) == 0; + boolean isDeleted = (flags & DELETION_MASK) != 0; + boolean isExpiring = (flags & EXPIRATION_MASK) != 0; + boolean useRowTimestamp = (flags & USE_ROW_TIMESTAMP) != 0; + boolean useRowTTL = (flags & USE_ROW_TTL) != 0; + + if (hasValue) + header.getType(column).skipValue(in); + + if (!useRowTimestamp) + in.readLong(); + + if (!useRowTTL && (isDeleted || isExpiring)) + in.readInt(); + + if (!useRowTTL && isExpiring) + in.readInt(); + + if (column.isComplex()) + column.cellPathSerializer().skip(in); + + return true; + } +} diff --git a/src/java/org/apache/cassandra/db/rows/WrappingRow.java b/src/java/org/apache/cassandra/db/rows/WrappingRow.java new file mode 100644 index 0000000000..5a0cc78f76 --- /dev/null +++ b/src/java/org/apache/cassandra/db/rows/WrappingRow.java @@ -0,0 +1,214 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.cassandra.db.rows; + +import java.util.Collections; +import java.util.Iterator; +import java.util.NoSuchElementException; + +import com.google.common.collect.UnmodifiableIterator; + +import org.apache.cassandra.config.ColumnDefinition; +import org.apache.cassandra.db.*; +import org.apache.cassandra.utils.SearchIterator; + +public abstract class WrappingRow extends AbstractRow +{ + protected Row wrapped; + + private final ReusableIterator cellIterator = new ReusableIterator(); + private final ReusableSearchIterator cellSearchIterator = new ReusableSearchIterator(); + + /** + * Apply some filtering/transformation on cells. This function + * can return {@code null} in which case the cell will be ignored. + */ + protected abstract Cell filterCell(Cell cell); + + protected DeletionTime filterDeletionTime(DeletionTime deletionTime) + { + return deletionTime; + } + + protected ColumnData filterColumnData(ColumnData data) + { + if (data.column().isComplex()) + { + Iterator cells = cellIterator.setTo(data.cells()); + DeletionTime dt = filterDeletionTime(data.complexDeletion()); + return cells == null && dt.isLive() + ? null + : new ColumnData(data.column(), null, cells == null ? Collections.emptyIterator(): cells, dt); + } + else + { + Cell filtered = filterCell(data.cell()); + return filtered == null ? null : new ColumnData(data.column(), filtered, null, null); + } + } + + public WrappingRow setTo(Row row) + { + this.wrapped = row; + return this; + } + + public Unfiltered.Kind kind() + { + return Unfiltered.Kind.ROW; + } + + public Clustering clustering() + { + return wrapped.clustering(); + } + + public Columns columns() + { + return wrapped.columns(); + } + + public LivenessInfo primaryKeyLivenessInfo() + { + return wrapped.primaryKeyLivenessInfo(); + } + + public DeletionTime deletion() + { + return wrapped.deletion(); + } + + public boolean hasComplexDeletion() + { + // Note that because cells can be filtered out/transformed through + // filterCell(), we can't rely on wrapped.hasComplexDeletion(). + for (int i = 0; i < columns().complexColumnCount(); i++) + if (!getDeletion(columns().getComplex(i)).isLive()) + return true; + return false; + } + + public Cell getCell(ColumnDefinition c) + { + Cell cell = wrapped.getCell(c); + return cell == null ? null : filterCell(cell); + } + + public Cell getCell(ColumnDefinition c, CellPath path) + { + Cell cell = wrapped.getCell(c, path); + return cell == null ? null : filterCell(cell); + } + + public Iterator getCells(ColumnDefinition c) + { + Iterator cells = wrapped.getCells(c); + if (cells == null) + return null; + + cellIterator.setTo(cells); + return cellIterator.hasNext() ? cellIterator : null; + } + + public DeletionTime getDeletion(ColumnDefinition c) + { + return filterDeletionTime(wrapped.getDeletion(c)); + } + + public Iterator iterator() + { + return cellIterator.setTo(wrapped.iterator()); + } + + public SearchIterator searchIterator() + { + return cellSearchIterator.setTo(wrapped.searchIterator()); + } + + public Row takeAlias() + { + boolean isCounter = columns().hasCounters(); + if (isStatic()) + { + StaticRow.Builder builder = StaticRow.builder(columns(), true, isCounter); + copyTo(builder); + return builder.build(); + } + else + { + ReusableRow copy = new ReusableRow(clustering().size(), columns(), true, isCounter); + copyTo(copy.writer()); + return copy; + } + } + + private class ReusableIterator extends UnmodifiableIterator + { + private Iterator iter; + private Cell next; + + public ReusableIterator setTo(Iterator iter) + { + this.iter = iter; + this.next = null; + return this; + } + + public boolean hasNext() + { + while (next == null && iter.hasNext()) + next = filterCell(iter.next()); + return next != null; + } + + public Cell next() + { + if (next == null && !hasNext()) + throw new NoSuchElementException(); + + Cell result = next; + next = null; + return result; + } + }; + + private class ReusableSearchIterator implements SearchIterator + { + private SearchIterator iter; + + public ReusableSearchIterator setTo(SearchIterator iter) + { + this.iter = iter; + return this; + } + + public boolean hasNext() + { + return iter.hasNext(); + } + + public ColumnData next(ColumnDefinition column) + { + ColumnData data = iter.next(column); + if (data == null) + return null; + + return filterColumnData(data); + } + }; +} diff --git a/src/java/org/apache/cassandra/db/rows/WrappingRowIterator.java b/src/java/org/apache/cassandra/db/rows/WrappingRowIterator.java new file mode 100644 index 0000000000..8847a4740f --- /dev/null +++ b/src/java/org/apache/cassandra/db/rows/WrappingRowIterator.java @@ -0,0 +1,79 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.cassandra.db.rows; + +import com.google.common.collect.UnmodifiableIterator; + +import org.apache.cassandra.config.CFMetaData; +import org.apache.cassandra.db.*; + +/** + * Abstract class to make writing atom iterators that wrap another iterator + * easier. By default, the wrapping iterator simply delegate every call to + * the wrapped iterator so concrete implementations will override some of the + * methods. + */ +public abstract class WrappingRowIterator extends UnmodifiableIterator implements RowIterator +{ + protected final RowIterator wrapped; + + protected WrappingRowIterator(RowIterator wrapped) + { + this.wrapped = wrapped; + } + + public CFMetaData metadata() + { + return wrapped.metadata(); + } + + public boolean isReverseOrder() + { + return wrapped.isReverseOrder(); + } + + public PartitionColumns columns() + { + return wrapped.columns(); + } + + public DecoratedKey partitionKey() + { + return wrapped.partitionKey(); + } + + public Row staticRow() + { + return wrapped.staticRow(); + } + + public boolean hasNext() + { + return wrapped.hasNext(); + } + + public Row next() + { + return wrapped.next(); + } + + public void close() + { + wrapped.close(); + } +} diff --git a/src/java/org/apache/cassandra/db/rows/WrappingUnfilteredRowIterator.java b/src/java/org/apache/cassandra/db/rows/WrappingUnfilteredRowIterator.java new file mode 100644 index 0000000000..680e502cbb --- /dev/null +++ b/src/java/org/apache/cassandra/db/rows/WrappingUnfilteredRowIterator.java @@ -0,0 +1,89 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.cassandra.db.rows; + +import com.google.common.collect.UnmodifiableIterator; + +import org.apache.cassandra.config.CFMetaData; +import org.apache.cassandra.db.*; + +/** + * Abstract class to make writing atom iterators that wrap another iterator + * easier. By default, the wrapping iterator simply delegate every call to + * the wrapped iterator so concrete implementations will override some of the + * methods. + */ +public abstract class WrappingUnfilteredRowIterator extends UnmodifiableIterator implements UnfilteredRowIterator +{ + protected final UnfilteredRowIterator wrapped; + + protected WrappingUnfilteredRowIterator(UnfilteredRowIterator wrapped) + { + this.wrapped = wrapped; + } + + public CFMetaData metadata() + { + return wrapped.metadata(); + } + + public PartitionColumns columns() + { + return wrapped.columns(); + } + + public boolean isReverseOrder() + { + return wrapped.isReverseOrder(); + } + + public DecoratedKey partitionKey() + { + return wrapped.partitionKey(); + } + + public DeletionTime partitionLevelDeletion() + { + return wrapped.partitionLevelDeletion(); + } + + public Row staticRow() + { + return wrapped.staticRow(); + } + + public boolean hasNext() + { + return wrapped.hasNext(); + } + + public Unfiltered next() + { + return wrapped.next(); + } + + public RowStats stats() + { + return wrapped.stats(); + } + + public void close() + { + wrapped.close(); + } +} diff --git a/src/java/org/apache/cassandra/dht/AbstractBounds.java b/src/java/org/apache/cassandra/dht/AbstractBounds.java index b5ffc22929..e295c68cf6 100644 --- a/src/java/org/apache/cassandra/dht/AbstractBounds.java +++ b/src/java/org/apache/cassandra/dht/AbstractBounds.java @@ -23,7 +23,7 @@ import java.io.Serializable; import java.util.List; import org.apache.cassandra.db.DecoratedKey; -import org.apache.cassandra.db.RowPosition; +import org.apache.cassandra.db.PartitionPosition; import org.apache.cassandra.db.TypeSizes; import org.apache.cassandra.db.marshal.AbstractType; import org.apache.cassandra.io.util.DataOutputPlus; @@ -34,8 +34,8 @@ public abstract class AbstractBounds> implements Seria private static final long serialVersionUID = 1L; public static final IPartitionerDependentSerializer> tokenSerializer = new AbstractBoundsSerializer(Token.serializer); - public static final IPartitionerDependentSerializer> rowPositionSerializer = - new AbstractBoundsSerializer(RowPosition.serializer); + public static final IPartitionerDependentSerializer> rowPositionSerializer = + new AbstractBoundsSerializer(PartitionPosition.serializer); private enum Type { @@ -112,6 +112,9 @@ public abstract class AbstractBounds> implements Seria protected abstract String getOpeningString(); protected abstract String getClosingString(); + public abstract boolean isStartInclusive(); + public abstract boolean isEndInclusive(); + public abstract AbstractBounds withNewRight(T newRight); public static class AbstractBoundsSerializer> implements IPartitionerDependentSerializer> diff --git a/src/java/org/apache/cassandra/dht/Bounds.java b/src/java/org/apache/cassandra/dht/Bounds.java index 4a5a7014ca..b569349698 100644 --- a/src/java/org/apache/cassandra/dht/Bounds.java +++ b/src/java/org/apache/cassandra/dht/Bounds.java @@ -20,7 +20,7 @@ package org.apache.cassandra.dht; import java.util.Collections; import java.util.List; -import org.apache.cassandra.db.RowPosition; +import org.apache.cassandra.db.PartitionPosition; import org.apache.cassandra.utils.Pair; /** @@ -102,12 +102,22 @@ public class Bounds> extends AbstractBounds return "]"; } + public boolean isStartInclusive() + { + return true; + } + + public boolean isEndInclusive() + { + return true; + } + /** * Compute a bounds of keys corresponding to a given bounds of token. */ - public static Bounds makeRowBounds(Token left, Token right) + public static Bounds makeRowBounds(Token left, Token right) { - return new Bounds(left.minKeyBound(), right.maxKeyBound()); + return new Bounds(left.minKeyBound(), right.maxKeyBound()); } public AbstractBounds withNewRight(T newRight) diff --git a/src/java/org/apache/cassandra/dht/ExcludingBounds.java b/src/java/org/apache/cassandra/dht/ExcludingBounds.java index 0d37e5c099..86af68d10e 100644 --- a/src/java/org/apache/cassandra/dht/ExcludingBounds.java +++ b/src/java/org/apache/cassandra/dht/ExcludingBounds.java @@ -90,6 +90,16 @@ public class ExcludingBounds> extends AbstractBounds withNewRight(T newRight) { return new ExcludingBounds(left, newRight); diff --git a/src/java/org/apache/cassandra/dht/IncludingExcludingBounds.java b/src/java/org/apache/cassandra/dht/IncludingExcludingBounds.java index e9e8e8e0de..446d0af442 100644 --- a/src/java/org/apache/cassandra/dht/IncludingExcludingBounds.java +++ b/src/java/org/apache/cassandra/dht/IncludingExcludingBounds.java @@ -89,6 +89,16 @@ public class IncludingExcludingBounds> extends Abstrac return ")"; } + public boolean isStartInclusive() + { + return true; + } + + public boolean isEndInclusive() + { + return false; + } + public AbstractBounds withNewRight(T newRight) { return new IncludingExcludingBounds(left, newRight); diff --git a/src/java/org/apache/cassandra/dht/Range.java b/src/java/org/apache/cassandra/dht/Range.java index cbf093c091..f99716ba49 100644 --- a/src/java/org/apache/cassandra/dht/Range.java +++ b/src/java/org/apache/cassandra/dht/Range.java @@ -21,7 +21,8 @@ import java.io.Serializable; import java.util.*; import org.apache.commons.lang3.ObjectUtils; -import org.apache.cassandra.db.RowPosition; + +import org.apache.cassandra.db.PartitionPosition; import org.apache.cassandra.utils.Pair; /** @@ -372,6 +373,16 @@ public class Range> extends AbstractBounds implemen return "]"; } + public boolean isStartInclusive() + { + return false; + } + + public boolean isEndInclusive() + { + return true; + } + public List asList() { ArrayList ret = new ArrayList(2); @@ -465,12 +476,12 @@ public class Range> extends AbstractBounds implemen /** * Compute a range of keys corresponding to a given range of token. */ - public static Range makeRowRange(Token left, Token right) + public static Range makeRowRange(Token left, Token right) { - return new Range(left.maxKeyBound(), right.maxKeyBound()); + return new Range(left.maxKeyBound(), right.maxKeyBound()); } - public static Range makeRowRange(Range tokenBounds) + public static Range makeRowRange(Range tokenBounds) { return makeRowRange(tokenBounds.left, tokenBounds.right); } diff --git a/src/java/org/apache/cassandra/dht/Token.java b/src/java/org/apache/cassandra/dht/Token.java index 0cc8a2dc00..c87b46b364 100644 --- a/src/java/org/apache/cassandra/dht/Token.java +++ b/src/java/org/apache/cassandra/dht/Token.java @@ -22,7 +22,7 @@ import java.io.IOException; import java.io.Serializable; import java.nio.ByteBuffer; -import org.apache.cassandra.db.RowPosition; +import org.apache.cassandra.db.PartitionPosition; import org.apache.cassandra.db.TypeSizes; import org.apache.cassandra.exceptions.ConfigurationException; import org.apache.cassandra.io.util.DataOutputPlus; @@ -142,7 +142,7 @@ public abstract class Token implements RingPosition, Serializable return (R)maxKeyBound(); } - public static class KeyBound implements RowPosition + public static class KeyBound implements PartitionPosition { private final Token token; public final boolean isMinimumBound; @@ -158,7 +158,7 @@ public abstract class Token implements RingPosition, Serializable return token; } - public int compareTo(RowPosition pos) + public int compareTo(PartitionPosition pos) { if (this == pos) return 0; @@ -188,9 +188,9 @@ public abstract class Token implements RingPosition, Serializable return getToken().isMinimum(); } - public RowPosition.Kind kind() + public PartitionPosition.Kind kind() { - return isMinimumBound ? RowPosition.Kind.MIN_BOUND : RowPosition.Kind.MAX_BOUND; + return isMinimumBound ? PartitionPosition.Kind.MIN_BOUND : PartitionPosition.Kind.MAX_BOUND; } @Override diff --git a/src/java/org/apache/cassandra/io/sstable/AbstractSSTableSimpleWriter.java b/src/java/org/apache/cassandra/io/sstable/AbstractSSTableSimpleWriter.java index b0a8f6b624..e7560c2092 100644 --- a/src/java/org/apache/cassandra/io/sstable/AbstractSSTableSimpleWriter.java +++ b/src/java/org/apache/cassandra/io/sstable/AbstractSSTableSimpleWriter.java @@ -17,42 +17,45 @@ */ package org.apache.cassandra.io.sstable; -import java.io.Closeable; import java.io.File; import java.io.FilenameFilter; import java.io.IOException; +import java.io.Closeable; import java.nio.ByteBuffer; import java.util.HashSet; import java.util.Set; import java.util.concurrent.atomic.AtomicInteger; +import com.google.common.base.Throwables; + import org.apache.cassandra.config.CFMetaData; import org.apache.cassandra.config.DatabaseDescriptor; import org.apache.cassandra.db.*; -import org.apache.cassandra.db.context.CounterContext; +import org.apache.cassandra.db.rows.RowStats; +import org.apache.cassandra.db.partitions.PartitionUpdate; import org.apache.cassandra.dht.IPartitioner; import org.apache.cassandra.io.sstable.format.SSTableFormat; import org.apache.cassandra.io.sstable.format.SSTableWriter; import org.apache.cassandra.service.ActiveRepairService; -import org.apache.cassandra.utils.CounterId; +import org.apache.cassandra.utils.FBUtilities; import org.apache.cassandra.utils.Pair; -public abstract class AbstractSSTableSimpleWriter implements Closeable +/** + * Base class for the sstable writers used by CQLSSTableWriter. + */ +abstract class AbstractSSTableSimpleWriter implements Closeable { protected final File directory; protected final CFMetaData metadata; - protected DecoratedKey currentKey; - protected ColumnFamily columnFamily; - protected ByteBuffer currentSuperColumn; - protected final CounterId counterid = CounterId.generate(); - private SSTableFormat.Type formatType = DatabaseDescriptor.getSSTableFormat(); + protected final PartitionColumns columns; + protected SSTableFormat.Type formatType = DatabaseDescriptor.getSSTableFormat(); protected static AtomicInteger generation = new AtomicInteger(0); - - public AbstractSSTableSimpleWriter(File directory, CFMetaData metadata, IPartitioner partitioner) + protected AbstractSSTableSimpleWriter(File directory, CFMetaData metadata, IPartitioner partitioner, PartitionColumns columns) { this.metadata = metadata; this.directory = directory; + this.columns = columns; DatabaseDescriptor.setPartitioner(partitioner); } @@ -61,12 +64,15 @@ public abstract class AbstractSSTableSimpleWriter implements Closeable this.formatType = type; } - protected SSTableWriter getWriter() + protected SSTableWriter createWriter() { - return SSTableWriter.create(createDescriptor(directory, metadata.ksName, metadata.cfName, formatType), 0, ActiveRepairService.UNREPAIRED_SSTABLE); + return SSTableWriter.create(createDescriptor(directory, metadata.ksName, metadata.cfName, formatType), + 0, + ActiveRepairService.UNREPAIRED_SSTABLE, + new SerializationHeader(metadata, columns, RowStats.NO_STATS)); } - protected static Descriptor createDescriptor(File directory, final String keyspace, final String columnFamily, final SSTableFormat.Type fmt) + private static Descriptor createDescriptor(File directory, final String keyspace, final String columnFamily, final SSTableFormat.Type fmt) { int maxGen = getNextGeneration(directory, columnFamily); return new Descriptor(directory, keyspace, columnFamily, maxGen + 1, Descriptor.Type.TEMP, fmt); @@ -102,108 +108,11 @@ public abstract class AbstractSSTableSimpleWriter implements Closeable } /** - * Start a new row whose key is {@code key}. - * @param key the row key + * Returns a PartitionUpdate suitable to write on this writer for the provided key. + * + * @param key they partition key for which the returned update will be. + * @return an update on partition {@code key} that is tied to this writer. */ - public void newRow(ByteBuffer key) throws IOException - { - if (currentKey != null && !columnFamily.isEmpty()) - writeRow(currentKey, columnFamily); - - currentKey = DatabaseDescriptor.getPartitioner().decorateKey(key); - columnFamily = getColumnFamily(); - } - - /** - * Start a new super column with name {@code name}. - * @param name the name for the super column - */ - public void newSuperColumn(ByteBuffer name) - { - if (!columnFamily.metadata().isSuper()) - throw new IllegalStateException("Cannot add a super column to a standard table"); - - currentSuperColumn = name; - } - - protected void addColumn(Cell cell) throws IOException - { - if (columnFamily.metadata().isSuper()) - { - if (currentSuperColumn == null) - throw new IllegalStateException("Trying to add a cell to a super column family, but no super cell has been started."); - - cell = cell.withUpdatedName(columnFamily.getComparator().makeCellName(currentSuperColumn, cell.name().toByteBuffer())); - } - columnFamily.addColumn(cell); - } - - /** - * Insert a new "regular" column to the current row (and super column if applicable). - * @param name the column name - * @param value the column value - * @param timestamp the column timestamp - */ - public void addColumn(ByteBuffer name, ByteBuffer value, long timestamp) throws IOException - { - addColumn(new BufferCell(metadata.comparator.cellFromByteBuffer(name), value, timestamp)); - } - - /** - * Insert a new expiring column to the current row (and super column if applicable). - * @param name the column name - * @param value the column value - * @param timestamp the column timestamp - * @param ttl the column time to live in seconds - * @param expirationTimestampMS the local expiration timestamp in milliseconds. This is the server time timestamp used for actually - * expiring the column, and as a consequence should be synchronized with the cassandra servers time. If {@code timestamp} represents - * the insertion time in microseconds (which is not required), this should be {@code (timestamp / 1000) + (ttl * 1000)}. - */ - public void addExpiringColumn(ByteBuffer name, ByteBuffer value, long timestamp, int ttl, long expirationTimestampMS) throws IOException - { - addColumn(new BufferExpiringCell(metadata.comparator.cellFromByteBuffer(name), value, timestamp, ttl, (int)(expirationTimestampMS / 1000))); - } - - /** - * Insert a new counter column to the current row (and super column if applicable). - * @param name the column name - * @param value the value of the counter - */ - public void addCounterColumn(ByteBuffer name, long value) throws IOException - { - addColumn(new BufferCounterCell(metadata.comparator.cellFromByteBuffer(name), - CounterContext.instance().createGlobal(counterid, 1L, value), - System.currentTimeMillis())); - } - - /** - * Package protected for use by AbstractCQLSSTableWriter. - * Not meant to be exposed publicly. - */ - ColumnFamily currentColumnFamily() - { - return columnFamily; - } - - /** - * Package protected for use by AbstractCQLSSTableWriter. - * Not meant to be exposed publicly. - */ - DecoratedKey currentKey() - { - return currentKey; - } - - /** - * Package protected for use by AbstractCQLSSTableWriter. - * Not meant to be exposed publicly. - */ - boolean shouldStartNewRow() throws IOException - { - return currentKey == null; - } - - protected abstract void writeRow(DecoratedKey key, ColumnFamily columnFamily) throws IOException; - - protected abstract ColumnFamily getColumnFamily() throws IOException; + abstract PartitionUpdate getUpdateFor(DecoratedKey key) throws IOException; } + diff --git a/src/java/org/apache/cassandra/io/sstable/CQLSSTableWriter.java b/src/java/org/apache/cassandra/io/sstable/CQLSSTableWriter.java index 4181ed050a..d3f3380957 100644 --- a/src/java/org/apache/cassandra/io/sstable/CQLSSTableWriter.java +++ b/src/java/org/apache/cassandra/io/sstable/CQLSSTableWriter.java @@ -25,18 +25,13 @@ import java.util.*; import com.google.common.collect.ImmutableMap; -import org.apache.cassandra.config.CFMetaData; -import org.apache.cassandra.config.Config; -import org.apache.cassandra.config.KSMetaData; -import org.apache.cassandra.config.Schema; +import org.apache.cassandra.config.*; import org.apache.cassandra.cql3.*; import org.apache.cassandra.cql3.statements.CreateTableStatement; import org.apache.cassandra.cql3.statements.ParsedStatement; import org.apache.cassandra.cql3.statements.UpdateStatement; -import org.apache.cassandra.db.ArrayBackedSortedColumns; -import org.apache.cassandra.db.Cell; -import org.apache.cassandra.db.ColumnFamily; -import org.apache.cassandra.db.composites.Composite; +import org.apache.cassandra.db.*; +import org.apache.cassandra.db.partitions.*; import org.apache.cassandra.db.marshal.AbstractType; import org.apache.cassandra.dht.IPartitioner; import org.apache.cassandra.dht.Murmur3Partitioner; @@ -208,26 +203,28 @@ public class CQLSSTableWriter implements Closeable QueryOptions options = QueryOptions.forInternalCalls(null, values); List keys = insert.buildPartitionKeyNames(options); - Composite clusteringPrefix = insert.createClusteringPrefix(options); + CBuilder clustering = insert.createClustering(options); long now = System.currentTimeMillis() * 1000; + // Note that we asks indexes to not validate values (the last 'false' arg below) because that triggers a 'Keyspace.open' + // and that forces a lot of initialization that we don't want. UpdateParameters params = new UpdateParameters(insert.cfm, options, insert.getTimestamp(now, options), insert.getTimeToLive(options), - Collections.emptyMap()); + Collections.emptyMap(), + false); try { for (ByteBuffer key : keys) { - if (writer.shouldStartNewRow() || !key.equals(writer.currentKey().getKey())) - writer.newRow(key); - insert.addUpdateForKey(writer.currentColumnFamily(), key, clusteringPrefix, params, false); + DecoratedKey dk = DatabaseDescriptor.getPartitioner().decorateKey(key); + insert.addUpdateForKey(writer.getUpdateFor(dk), clustering, params); } return this; } - catch (BufferedWriter.SyncException e) + catch (SSTableSimpleUnsortedWriter.SyncException e) { // If we use a BufferedWriter and had a problem writing to disk, the IOException has been // wrapped in a SyncException (see BufferedWriter below). We want to extract that IOE. @@ -348,7 +345,7 @@ public class CQLSSTableWriter implements Closeable { synchronized (CQLSSTableWriter.class) { - this.schema = getStatement(schema, CreateTableStatement.class, "CREATE TABLE").left.getCFMetaData().rebuild(); + this.schema = getStatement(schema, CreateTableStatement.class, "CREATE TABLE").left.getCFMetaData(); // We need to register the keyspace/table metadata through Schema, otherwise we won't be able to properly // build the insert statement in using(). @@ -373,7 +370,7 @@ public class CQLSSTableWriter implements Closeable /** * Creates the keyspace with the specified table. * - * @param the table the table that must be created. + * @param table the table that must be created. */ private static void createKeyspaceWithTable(CFMetaData table) { @@ -520,8 +517,8 @@ public class CQLSSTableWriter implements Closeable throw new IllegalStateException("No insert statement specified, you should provide an insert statement through using()"); AbstractSSTableSimpleWriter writer = sorted - ? new SSTableSimpleWriter(directory, schema, partitioner) - : new BufferedWriter(directory, schema, partitioner, bufferSizeInMB); + ? new SSTableSimpleWriter(directory, schema, partitioner, insert.updatedColumns()) + : new SSTableSimpleUnsortedWriter(directory, schema, partitioner, insert.updatedColumns(), bufferSizeInMB); if (formatType != null) writer.setSSTableFormatType(formatType); @@ -529,81 +526,4 @@ public class CQLSSTableWriter implements Closeable return new CQLSSTableWriter(writer, insert, boundNames); } } - - /** - * CQLSSTableWriter doesn't use the method addColumn() from AbstractSSTableSimpleWriter. - * Instead, it adds cells directly to the ColumnFamily the latter exposes. But this means - * that the sync() method of SSTableSimpleUnsortedWriter is not called (at least not for - * each CQL row, so adding many rows to the same partition can buffer too much data in - * memory - #7360). So we create a slightly modified SSTableSimpleUnsortedWriter that uses - * a tweaked ColumnFamily object that calls back the proper method after each added cell - * so we sync when we should. - */ - private static class BufferedWriter extends SSTableSimpleUnsortedWriter - { - private boolean needsSync = false; - - public BufferedWriter(File directory, CFMetaData metadata, IPartitioner partitioner, long bufferSizeInMB) - { - super(directory, metadata, partitioner, bufferSizeInMB); - } - - @Override - protected ColumnFamily createColumnFamily() - { - return new ArrayBackedSortedColumns(metadata, false) - { - @Override - public void addColumn(Cell cell) - { - super.addColumn(cell); - try - { - countColumn(cell); - } - catch (IOException e) - { - // addColumn does not throw IOException but we want to report this to the user, - // so wrap it in a temporary RuntimeException that we'll catch in rawAddRow above. - throw new SyncException(e); - } - } - }; - } - - @Override - protected void replaceColumnFamily() throws IOException - { - needsSync = true; - } - - /** - * If we have marked that the column family is being replaced, when we start the next row, - * we should sync out the previous partition and create a new row based on the current value. - */ - @Override - boolean shouldStartNewRow() throws IOException - { - if (needsSync) - { - needsSync = false; - super.sync(); - return true; - } - return super.shouldStartNewRow(); - } - - protected void addColumn(Cell cell) throws IOException - { - throw new UnsupportedOperationException(); - } - - static class SyncException extends RuntimeException - { - SyncException(IOException ioe) - { - super(ioe); - } - } - } } diff --git a/src/java/org/apache/cassandra/io/sstable/ColumnNameHelper.java b/src/java/org/apache/cassandra/io/sstable/ColumnNameHelper.java deleted file mode 100644 index 846634a121..0000000000 --- a/src/java/org/apache/cassandra/io/sstable/ColumnNameHelper.java +++ /dev/null @@ -1,241 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.apache.cassandra.io.sstable; - -import java.nio.ByteBuffer; -import java.util.ArrayList; -import java.util.List; - -import org.apache.cassandra.db.composites.CellNameType; -import org.apache.cassandra.db.composites.Composite; -import org.apache.cassandra.db.marshal.AbstractType; - -import static org.apache.cassandra.utils.ByteBufferUtil.minimalBufferFor; - -public class ColumnNameHelper -{ - private static List maybeGrow(List l, int size) - { - if (l.size() >= size) - return l; - - List nl = new ArrayList<>(size); - nl.addAll(l); - for (int i = l.size(); i < size; i++) - nl.add(null); - return nl; - } - - private static List getComponents(Composite prefix, int size) - { - List l = new ArrayList<>(size); - for (int i = 0; i < size; i++) - l.add(prefix.get(i)); - return l; - } - - /** - * finds the max cell name component(s) - * - * Note that this method *can modify maxSeen*. - * - * @param maxSeen the max columns seen so far - * @param candidate the candidate column(s) - * @param comparator the comparator to use - * @return a list with the max column(s) - */ - public static List maxComponents(List maxSeen, Composite candidate, CellNameType comparator) - { - // For a cell name, no reason to look more than the clustering prefix - // (and comparing the collection element would actually crash) - int size = Math.min(candidate.size(), comparator.clusteringPrefixSize()); - - if (maxSeen.isEmpty()) - return getComponents(candidate, size); - - // In most case maxSeen is big enough to hold the result so update it in place in those cases - maxSeen = maybeGrow(maxSeen, size); - - for (int i = 0; i < size; i++) - maxSeen.set(i, max(maxSeen.get(i), candidate.get(i), comparator.subtype(i))); - - return maxSeen; - } - - /** - * finds the min cell name component(s) - * - * Note that this method *can modify maxSeen*. - * - * @param minSeen the max columns seen so far - * @param candidate the candidate column(s) - * @param comparator the comparator to use - * @return a list with the min column(s) - */ - public static List minComponents(List minSeen, Composite candidate, CellNameType comparator) - { - // For a cell name, no reason to look more than the clustering prefix - // (and comparing the collection element would actually crash) - int size = Math.min(candidate.size(), comparator.clusteringPrefixSize()); - - if (minSeen.isEmpty()) - return getComponents(candidate, size); - - // In most case maxSeen is big enough to hold the result so update it in place in those cases - minSeen = maybeGrow(minSeen, size); - - for (int i = 0; i < size; i++) - minSeen.set(i, min(minSeen.get(i), candidate.get(i), comparator.subtype(i))); - - return minSeen; - } - - /** - * return the min column - * - * note that comparator should not be of CompositeType! - * - * @param b1 lhs - * @param b2 rhs - * @param comparator the comparator to use - * @return the smallest column according to comparator - */ - private static ByteBuffer min(ByteBuffer b1, ByteBuffer b2, AbstractType comparator) - { - if (b1 == null) - return b2; - if (b2 == null) - return b1; - - if (comparator.compare(b1, b2) >= 0) - return b2; - return b1; - } - - /** - * return the max column - * - * note that comparator should not be of CompositeType! - * - * @param b1 lhs - * @param b2 rhs - * @param comparator the comparator to use - * @return the biggest column according to comparator - */ - private static ByteBuffer max(ByteBuffer b1, ByteBuffer b2, AbstractType comparator) - { - if (b1 == null) - return b2; - if (b2 == null) - return b1; - - if (comparator.compare(b1, b2) >= 0) - return b1; - return b2; - } - - /** - * Merge 2 lists of min cell name components. - * - * @param minColumnNames lhs - * @param candidates rhs - * @param comparator comparator to use - * @return a list with smallest column names according to (sub)comparator - */ - public static List mergeMin(List minColumnNames, List candidates, CellNameType comparator) - { - if (minColumnNames.isEmpty()) - return minimalBuffersFor(candidates); - - if (candidates.isEmpty()) - return minColumnNames; - - List biggest = minColumnNames.size() > candidates.size() ? minColumnNames : candidates; - List smallest = minColumnNames.size() > candidates.size() ? candidates : minColumnNames; - - // We want to always copy the smallest list, and maybeGrow does it only if it's actually smaller - List retList = smallest.size() == biggest.size() - ? new ArrayList<>(smallest) - : maybeGrow(smallest, biggest.size()); - - for (int i = 0; i < biggest.size(); i++) - retList.set(i, minimalBufferFor(min(retList.get(i), biggest.get(i), comparator.subtype(i)))); - - return retList; - } - - private static List minimalBuffersFor(List candidates) - { - List minimalBuffers = new ArrayList(candidates.size()); - for (ByteBuffer byteBuffer : candidates) - minimalBuffers.add(minimalBufferFor(byteBuffer)); - return minimalBuffers; - } - - /** - * Merge 2 lists of max cell name components. - * - * @param maxColumnNames lhs - * @param candidates rhs - * @param comparator comparator to use - * @return a list with biggest column names according to (sub)comparator - */ - public static List mergeMax(List maxColumnNames, List candidates, CellNameType comparator) - { - if (maxColumnNames.isEmpty()) - return minimalBuffersFor(candidates); - - if (candidates.isEmpty()) - return maxColumnNames; - - List biggest = maxColumnNames.size() > candidates.size() ? maxColumnNames : candidates; - List smallest = maxColumnNames.size() > candidates.size() ? candidates : maxColumnNames; - - // We want to always copy the smallest list, and maybeGrow does it only if it's actually smaller - List retList = smallest.size() == biggest.size() - ? new ArrayList<>(smallest) - : maybeGrow(smallest, biggest.size()); - - for (int i = 0; i < biggest.size(); i++) - retList.set(i, minimalBufferFor(max(retList.get(i), biggest.get(i), comparator.subtype(i)))); - - return retList; - } - - /** - * Checks if the given min/max column names could overlap (i.e they could share some column names based on the max/min column names in the sstables) - */ - public static boolean overlaps(List minColumnNames1, List maxColumnNames1, List minColumnNames2, List maxColumnNames2, CellNameType comparator) - { - if (minColumnNames1.isEmpty() || maxColumnNames1.isEmpty() || minColumnNames2.isEmpty() || maxColumnNames2.isEmpty()) - return true; - - return !(compare(maxColumnNames1, minColumnNames2, comparator) < 0 || compare(minColumnNames1, maxColumnNames2, comparator) > 0); - } - - private static int compare(List columnNames1, List columnNames2, CellNameType comparator) - { - for (int i = 0; i < Math.min(columnNames1.size(), columnNames2.size()); i++) - { - int cmp = comparator.subtype(i).compare(columnNames1.get(i), columnNames2.get(i)); - if (cmp != 0) - return cmp; - } - return 0; - } -} diff --git a/src/java/org/apache/cassandra/io/sstable/ColumnStats.java b/src/java/org/apache/cassandra/io/sstable/ColumnStats.java deleted file mode 100644 index a1cb199391..0000000000 --- a/src/java/org/apache/cassandra/io/sstable/ColumnStats.java +++ /dev/null @@ -1,165 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.apache.cassandra.io.sstable; - -import java.nio.ByteBuffer; -import java.util.List; - -import org.apache.cassandra.utils.StreamingHistogram; - -/** - * ColumnStats holds information about the columns for one row inside sstable - */ -public class ColumnStats -{ - /** how many columns are there in the row */ - public final int columnCount; - - /** the largest (client-supplied) timestamp in the row */ - public final long minTimestamp; - public final long maxTimestamp; - public final int maxLocalDeletionTime; - /** histogram of tombstone drop time */ - public final StreamingHistogram tombstoneHistogram; - - /** max and min column names according to comparator */ - public final List minColumnNames; - public final List maxColumnNames; - - public final boolean hasLegacyCounterShards; - - public ColumnStats(int columnCount, - long minTimestamp, - long maxTimestamp, - int maxLocalDeletionTime, - StreamingHistogram tombstoneHistogram, - List minColumnNames, - List maxColumnNames, - boolean hasLegacyCounterShards) - { - this.minTimestamp = minTimestamp; - this.maxTimestamp = maxTimestamp; - this.maxLocalDeletionTime = maxLocalDeletionTime; - this.columnCount = columnCount; - this.tombstoneHistogram = tombstoneHistogram; - this.minColumnNames = minColumnNames; - this.maxColumnNames = maxColumnNames; - this.hasLegacyCounterShards = hasLegacyCounterShards; - } - - // We use explicit classes for ints and longs instead of generics to avoid boxing and unboxing (See CASSANDRA-8109) - public static class MinLongTracker - { - private final long defaultValue; - private boolean isSet = false; - private long value; - - public MinLongTracker(long defaultValue) - { - this.defaultValue = defaultValue; - } - - public void update(long value) - { - if (!isSet) - { - this.value = value; - isSet = true; - } - else - { - if (value < this.value) - this.value = value; - } - } - - public long get() - { - if (isSet) - return value; - return defaultValue; - } - } - - public static class MaxLongTracker - { - private final long defaultValue; - private boolean isSet = false; - private long value; - - public MaxLongTracker(long defaultValue) - { - this.defaultValue = defaultValue; - } - - public void update(long value) - { - if (!isSet) - { - this.value = value; - isSet = true; - } - else - { - if (value >this.value) - this.value = value; - } - } - - public long get() - { - if (isSet) - return value; - return defaultValue; - } - } - - public static class MaxIntTracker - { - private final int defaultValue; - private boolean isSet = false; - private int value; - - public MaxIntTracker(int defaultValue) - { - this.defaultValue = defaultValue; - } - - public void update(int value) - { - if (!isSet) - { - this.value = value; - isSet = true; - } - else - { - if (value > this.value) - this.value = value; - } - } - - public int get() - { - if (isSet) - return value; - return defaultValue; - } - } - -} diff --git a/src/java/org/apache/cassandra/io/sstable/CorruptSSTableException.java b/src/java/org/apache/cassandra/io/sstable/CorruptSSTableException.java index a71daaf44d..0fe316dd2f 100644 --- a/src/java/org/apache/cassandra/io/sstable/CorruptSSTableException.java +++ b/src/java/org/apache/cassandra/io/sstable/CorruptSSTableException.java @@ -25,7 +25,7 @@ public class CorruptSSTableException extends RuntimeException public CorruptSSTableException(Exception cause, File path) { - super(cause); + super("Corrupted: " + path, cause); this.path = path; } diff --git a/src/java/org/apache/cassandra/io/sstable/Descriptor.java b/src/java/org/apache/cassandra/io/sstable/Descriptor.java index 9f259fe060..5ab99e701a 100644 --- a/src/java/org/apache/cassandra/io/sstable/Descriptor.java +++ b/src/java/org/apache/cassandra/io/sstable/Descriptor.java @@ -32,6 +32,7 @@ import org.apache.cassandra.io.sstable.format.Version; import org.apache.cassandra.io.sstable.metadata.IMetadataSerializer; import org.apache.cassandra.io.sstable.metadata.LegacyMetadataSerializer; import org.apache.cassandra.io.sstable.metadata.MetadataSerializer; +import org.apache.cassandra.net.MessagingService; import org.apache.cassandra.utils.Pair; import static org.apache.cassandra.io.sstable.Component.separator; @@ -45,7 +46,6 @@ import static org.apache.cassandra.io.sstable.Component.separator; */ public class Descriptor { - public static enum Type { TEMP("tmp", true), TEMPLINK("tmplink", true), FINAL(null, false); @@ -58,7 +58,6 @@ public class Descriptor } } - public final File directory; /** version has the following format: [a-z]+ */ public final Version version; diff --git a/src/java/org/apache/cassandra/io/sstable/ISSTableScanner.java b/src/java/org/apache/cassandra/io/sstable/ISSTableScanner.java index b80bd87ec1..7063057170 100644 --- a/src/java/org/apache/cassandra/io/sstable/ISSTableScanner.java +++ b/src/java/org/apache/cassandra/io/sstable/ISSTableScanner.java @@ -19,14 +19,13 @@ package org.apache.cassandra.io.sstable; -import org.apache.cassandra.db.columniterator.OnDiskAtomIterator; -import org.apache.cassandra.utils.CloseableIterator; +import org.apache.cassandra.db.partitions.UnfilteredPartitionIterator; /** * An ISSTableScanner is an abstraction allowing multiple SSTableScanners to be * chained together under the hood. See LeveledCompactionStrategy.getScanners. */ -public interface ISSTableScanner extends CloseableIterator +public interface ISSTableScanner extends UnfilteredPartitionIterator { public long getLengthInBytes(); public long getCurrentPosition(); diff --git a/src/java/org/apache/cassandra/io/sstable/IndexHelper.java b/src/java/org/apache/cassandra/io/sstable/IndexHelper.java index 3d304c5360..d19c8f7290 100644 --- a/src/java/org/apache/cassandra/io/sstable/IndexHelper.java +++ b/src/java/org/apache/cassandra/io/sstable/IndexHelper.java @@ -22,10 +22,10 @@ import java.util.Collections; import java.util.Comparator; import java.util.List; -import org.apache.cassandra.db.composites.CType; -import org.apache.cassandra.db.composites.Composite; -import org.apache.cassandra.db.TypeSizes; +import org.apache.cassandra.config.CFMetaData; +import org.apache.cassandra.db.*; import org.apache.cassandra.io.ISerializer; +import org.apache.cassandra.io.sstable.format.Version; import org.apache.cassandra.io.util.DataOutputPlus; import org.apache.cassandra.io.util.FileDataInput; import org.apache.cassandra.io.util.FileUtils; @@ -67,35 +67,23 @@ public class IndexHelper /** * The index of the IndexInfo in which a scan starting with @name should begin. * - * @param name - * name of the index - * - * @param indexList - * list of the indexInfo objects - * - * @param comparator - * comparator type - * - * @param reversed - * is name reversed + * @param name name to search for + * @param indexList list of the indexInfo objects + * @param comparator the comparator to use + * @param reversed whether or not the search is reversed, i.e. we scan forward or backward from name + * @param lastIndex where to start the search from in indexList * * @return int index */ - public static int indexFor(Composite name, List indexList, CType comparator, boolean reversed, int lastIndex) + public static int indexFor(ClusteringPrefix name, List indexList, ClusteringComparator comparator, boolean reversed, int lastIndex) { - if (name.isEmpty()) - return lastIndex >= 0 ? lastIndex : reversed ? indexList.size() - 1 : 0; - - if (lastIndex >= indexList.size()) - return -1; - - IndexInfo target = new IndexInfo(name, name, 0, 0); + IndexInfo target = new IndexInfo(name, name, 0, 0, null); /* Take the example from the unit test, and say your index looks like this: [0..5][10..15][20..25] and you look for the slice [13..17]. - When doing forward slice, we we doing a binary search comparing 13 (the start of the query) + When doing forward slice, we are doing a binary search comparing 13 (the start of the query) to the lastName part of the index slot. You'll end up with the "first" slot, going from left to right, that may contain the start. @@ -105,81 +93,117 @@ public class IndexHelper */ int startIdx = 0; List toSearch = indexList; - if (lastIndex >= 0) + if (reversed) { - if (reversed) + if (lastIndex < indexList.size() - 1) { toSearch = indexList.subList(0, lastIndex + 1); } - else + } + else + { + if (lastIndex > 0) { startIdx = lastIndex; toSearch = indexList.subList(lastIndex, indexList.size()); } } - int index = Collections.binarySearch(toSearch, target, getComparator(comparator, reversed)); + int index = Collections.binarySearch(toSearch, target, comparator.indexComparator(reversed)); return startIdx + (index < 0 ? -index - (reversed ? 2 : 1) : index); } - public static Comparator getComparator(final CType nameComparator, boolean reversed) - { - return reversed ? nameComparator.indexReverseComparator() : nameComparator.indexComparator(); - } - public static class IndexInfo { - private static final long EMPTY_SIZE = ObjectSizes.measure(new IndexInfo(null, null, 0, 0)); + private static final long EMPTY_SIZE = ObjectSizes.measure(new IndexInfo(null, null, 0, 0, null)); public final long width; - public final Composite lastName; - public final Composite firstName; + public final ClusteringPrefix lastName; + public final ClusteringPrefix firstName; public final long offset; - public IndexInfo(Composite firstName, Composite lastName, long offset, long width) + // If at the end of the index block there is an open range tombstone marker, this marker + // deletion infos. null otherwise. + public final DeletionTime endOpenMarker; + + public IndexInfo(ClusteringPrefix firstName, + ClusteringPrefix lastName, + long offset, + long width, + DeletionTime endOpenMarker) { this.firstName = firstName; this.lastName = lastName; this.offset = offset; this.width = width; + this.endOpenMarker = endOpenMarker; } - public static class Serializer implements ISerializer + public static class Serializer { - private final CType type; + private final CFMetaData metadata; + private final Version version; - public Serializer(CType type) + public Serializer(CFMetaData metadata, Version version) { - this.type = type; + this.metadata = metadata; + this.version = version; } - public void serialize(IndexInfo info, DataOutputPlus out) throws IOException + public void serialize(IndexInfo info, DataOutputPlus out, SerializationHeader header) throws IOException { - type.serializer().serialize(info.firstName, out); - type.serializer().serialize(info.lastName, out); + ISerializer clusteringSerializer = metadata.serializers().clusteringPrefixSerializer(version, header); + clusteringSerializer.serialize(info.firstName, out); + clusteringSerializer.serialize(info.lastName, out); out.writeLong(info.offset); out.writeLong(info.width); + + if (version.storeRows()) + { + out.writeBoolean(info.endOpenMarker != null); + if (info.endOpenMarker != null) + DeletionTime.serializer.serialize(info.endOpenMarker, out); + } } - public IndexInfo deserialize(DataInput in) throws IOException + public IndexInfo deserialize(DataInput in, SerializationHeader header) throws IOException { - return new IndexInfo(type.serializer().deserialize(in), - type.serializer().deserialize(in), - in.readLong(), - in.readLong()); + ISerializer clusteringSerializer = metadata.serializers().clusteringPrefixSerializer(version, header); + + ClusteringPrefix firstName = clusteringSerializer.deserialize(in); + ClusteringPrefix lastName = clusteringSerializer.deserialize(in); + long offset = in.readLong(); + long width = in.readLong(); + DeletionTime endOpenMarker = version.storeRows() && in.readBoolean() + ? DeletionTime.serializer.deserialize(in) + : null; + + return new IndexInfo(firstName, lastName, offset, width, endOpenMarker); } - public long serializedSize(IndexInfo info, TypeSizes typeSizes) + public long serializedSize(IndexInfo info, SerializationHeader header, TypeSizes typeSizes) { - return type.serializer().serializedSize(info.firstName, typeSizes) - + type.serializer().serializedSize(info.lastName, typeSizes) - + typeSizes.sizeof(info.offset) - + typeSizes.sizeof(info.width); + ISerializer clusteringSerializer = metadata.serializers().clusteringPrefixSerializer(version, header); + long size = clusteringSerializer.serializedSize(info.firstName, typeSizes) + + clusteringSerializer.serializedSize(info.lastName, typeSizes) + + typeSizes.sizeof(info.offset) + + typeSizes.sizeof(info.width); + + if (version.storeRows()) + { + size += typeSizes.sizeof(info.endOpenMarker != null); + if (info.endOpenMarker != null) + size += DeletionTime.serializer.serializedSize(info.endOpenMarker, typeSizes); + } + return size; } } public long unsharedHeapSize() { - return EMPTY_SIZE + firstName.unsharedHeapSize() + lastName.unsharedHeapSize(); + return EMPTY_SIZE + + firstName.unsharedHeapSize() + + lastName.unsharedHeapSize() + + (endOpenMarker == null ? 0 : endOpenMarker.unsharedHeapSize()); } } } diff --git a/src/java/org/apache/cassandra/io/sstable/IndexSummary.java b/src/java/org/apache/cassandra/io/sstable/IndexSummary.java index 7df734928d..90c5b0ea06 100644 --- a/src/java/org/apache/cassandra/io/sstable/IndexSummary.java +++ b/src/java/org/apache/cassandra/io/sstable/IndexSummary.java @@ -18,8 +18,6 @@ package org.apache.cassandra.io.sstable; import java.io.DataInputStream; -import java.io.File; -import java.io.FileOutputStream; import java.io.IOException; import java.nio.ByteBuffer; import java.nio.ByteOrder; @@ -28,10 +26,9 @@ import org.slf4j.Logger; import org.slf4j.LoggerFactory; import org.apache.cassandra.db.DecoratedKey; -import org.apache.cassandra.db.RowPosition; +import org.apache.cassandra.db.PartitionPosition; import org.apache.cassandra.dht.IPartitioner; import org.apache.cassandra.io.util.*; -import org.apache.cassandra.utils.ByteBufferUtil; import org.apache.cassandra.utils.FBUtilities; import org.apache.cassandra.utils.concurrent.WrappedSharedCloseable; import org.apache.cassandra.utils.memory.MemoryUtil; @@ -110,7 +107,7 @@ public class IndexSummary extends WrappedSharedCloseable // binary search is notoriously more difficult to get right than it looks; this is lifted from // Harmony's Collections implementation - public int binarySearch(RowPosition key) + public int binarySearch(PartitionPosition key) { // We will be comparing non-native Keys, so use a buffer with appropriate byte order ByteBuffer hollow = MemoryUtil.getHollowDirectByteBuffer().order(ByteOrder.BIG_ENDIAN); diff --git a/src/java/org/apache/cassandra/io/sstable/ReducingKeyIterator.java b/src/java/org/apache/cassandra/io/sstable/ReducingKeyIterator.java index e5f61b2055..f1e01f215c 100644 --- a/src/java/org/apache/cassandra/io/sstable/ReducingKeyIterator.java +++ b/src/java/org/apache/cassandra/io/sstable/ReducingKeyIterator.java @@ -57,7 +57,7 @@ public class ReducingKeyIterator implements CloseableIterator return true; } - public void reduce(DecoratedKey current) + public void reduce(int idx, DecoratedKey current) { reduced = current; } diff --git a/src/java/org/apache/cassandra/io/sstable/SSTableIdentityIterator.java b/src/java/org/apache/cassandra/io/sstable/SSTableIdentityIterator.java index 17f9a8d33e..70ab99c0bd 100644 --- a/src/java/org/apache/cassandra/io/sstable/SSTableIdentityIterator.java +++ b/src/java/org/apache/cassandra/io/sstable/SSTableIdentityIterator.java @@ -18,31 +18,24 @@ package org.apache.cassandra.io.sstable; import java.io.*; -import java.util.Iterator; + +import com.google.common.collect.AbstractIterator; import org.apache.cassandra.config.CFMetaData; -import org.apache.cassandra.config.DatabaseDescriptor; import org.apache.cassandra.db.*; -import org.apache.cassandra.db.columniterator.OnDiskAtomIterator; +import org.apache.cassandra.db.rows.*; import org.apache.cassandra.io.sstable.format.SSTableReader; -import org.apache.cassandra.io.sstable.format.Version; -import org.apache.cassandra.io.util.FileDataInput; import org.apache.cassandra.io.util.RandomAccessReader; -import org.apache.cassandra.serializers.MarshalException; - public class SSTableIdentityIterator implements Comparable, OnDiskAtomIterator +public class SSTableIdentityIterator extends AbstractIterator implements Comparable, UnfilteredRowIterator { + private final SSTableReader sstable; private final DecoratedKey key; - private final DataInput in; - public final ColumnSerializer.Flag flag; - - private final ColumnFamily columnFamily; - private final Iterator atomIterator; - private final boolean validateColumns; + private final DeletionTime partitionLevelDeletion; private final String filename; - // Not every SSTableIdentifyIterator is attached to a sstable, so this can be null. - private final SSTableReader sstable; + private final SSTableSimpleIterator iterator; + private final Row staticRow; /** * Used to iterate through the columns of a row. @@ -52,80 +45,66 @@ import org.apache.cassandra.serializers.MarshalException; */ public SSTableIdentityIterator(SSTableReader sstable, RandomAccessReader file, DecoratedKey key) { - this(sstable, file, key, false); - } - - /** - * Used to iterate through the columns of a row. - * @param sstable SSTable we are reading ffrom. - * @param file Reading using this file. - * @param key Key of this row. - * @param checkData if true, do its best to deserialize and check the coherence of row data - */ - public SSTableIdentityIterator(SSTableReader sstable, RandomAccessReader file, DecoratedKey key, boolean checkData) - { - this(sstable.metadata, file, file.getPath(), key, checkData, sstable, ColumnSerializer.Flag.LOCAL); - } - - // sstable may be null *if* checkData is false - // If it is null, we assume the data is in the current file format - private SSTableIdentityIterator(CFMetaData metadata, - FileDataInput in, - String filename, - DecoratedKey key, - boolean checkData, - SSTableReader sstable, - ColumnSerializer.Flag flag) - { - assert !checkData || (sstable != null); - this.in = in; - this.filename = filename; - this.key = key; - this.flag = flag; - this.validateColumns = checkData; this.sstable = sstable; - - Version dataVersion = sstable == null ? DatabaseDescriptor.getSSTableFormat().info.getLatestVersion() : sstable.descriptor.version; - int expireBefore = (int) (System.currentTimeMillis() / 1000); - columnFamily = ArrayBackedSortedColumns.factory.create(metadata); + this.filename = file.getPath(); + this.key = key; try { - columnFamily.delete(DeletionTime.serializer.deserialize(in)); - atomIterator = columnFamily.metadata().getOnDiskIterator(in, flag, expireBefore, dataVersion); + this.partitionLevelDeletion = DeletionTime.serializer.deserialize(file); + SerializationHelper helper = new SerializationHelper(sstable.descriptor.version.correspondingMessagingVersion(), SerializationHelper.Flag.LOCAL); + this.iterator = SSTableSimpleIterator.create(sstable.metadata, file, sstable.header, helper, partitionLevelDeletion); + this.staticRow = iterator.readStaticRow(); } catch (IOException e) { - if (sstable != null) - sstable.markSuspect(); + sstable.markSuspect(); throw new CorruptSSTableException(e, filename); } } - public DecoratedKey getKey() + public CFMetaData metadata() + { + return sstable.metadata; + } + + public PartitionColumns columns() + { + return metadata().partitionColumns(); + } + + public boolean isReverseOrder() + { + return false; + } + + public DecoratedKey partitionKey() { return key; } - public ColumnFamily getColumnFamily() + public DeletionTime partitionLevelDeletion() { - return columnFamily; + return partitionLevelDeletion; } - public boolean hasNext() + public Row staticRow() + { + return staticRow; + } + + protected Unfiltered computeNext() { try { - return atomIterator.hasNext(); + return iterator.hasNext() ? iterator.next() : endOfData(); } catch (IOError e) { - // catch here b/c atomIterator is an AbstractIterator; hasNext reads the value if (e.getCause() instanceof IOException) { - if (sstable != null) - sstable.markSuspect(); - throw new CorruptSSTableException((IOException)e.getCause(), filename); + sstable.markSuspect(); + throw new CorruptSSTableException((Exception)e.getCause(), filename); } else { @@ -134,26 +113,6 @@ import org.apache.cassandra.serializers.MarshalException; } } - public OnDiskAtom next() - { - try - { - OnDiskAtom atom = atomIterator.next(); - if (validateColumns) - atom.validateFields(columnFamily.metadata()); - return atom; - } - catch (MarshalException me) - { - throw new CorruptSSTableException(me, filename); - } - } - - public void remove() - { - throw new UnsupportedOperationException(); - } - public void close() { // creator is responsible for closing file when finished @@ -161,16 +120,14 @@ import org.apache.cassandra.serializers.MarshalException; public String getPath() { - // if input is from file, then return that path, otherwise it's from streaming - if (in instanceof RandomAccessReader) - { - RandomAccessReader file = (RandomAccessReader) in; - return file.getPath(); - } - else - { - throw new UnsupportedOperationException(); - } + return filename; + } + + public RowStats stats() + { + // We could return sstable.header.stats(), but this may not be as accurate than the actual sstable stats (see + // SerializationHeader.make() for details) so we use the latter instead. + return new RowStats(sstable.getMinTimestamp(), sstable.getMinLocalDeletionTime(), sstable.getMinTTL(), sstable.getAvgColumnSetPerRow()); } public int compareTo(SSTableIdentityIterator o) diff --git a/src/java/org/apache/cassandra/io/sstable/SSTableRewriter.java b/src/java/org/apache/cassandra/io/sstable/SSTableRewriter.java index 9497bf3b21..260e137903 100644 --- a/src/java/org/apache/cassandra/io/sstable/SSTableRewriter.java +++ b/src/java/org/apache/cassandra/io/sstable/SSTableRewriter.java @@ -26,7 +26,7 @@ import org.apache.cassandra.config.DatabaseDescriptor; import org.apache.cassandra.db.ColumnFamilyStore; import org.apache.cassandra.db.DecoratedKey; import org.apache.cassandra.db.RowIndexEntry; -import org.apache.cassandra.db.compaction.AbstractCompactedRow; +import org.apache.cassandra.db.rows.UnfilteredRowIterator; import org.apache.cassandra.io.sstable.format.SSTableReader; import org.apache.cassandra.io.sstable.format.SSTableWriter; import org.apache.cassandra.db.lifecycle.LifecycleTransaction; @@ -108,42 +108,36 @@ public class SSTableRewriter extends Transactional.AbstractTransactional impleme return writer; } - public RowIndexEntry append(AbstractCompactedRow row) + public RowIndexEntry append(UnfilteredRowIterator partition) { // we do this before appending to ensure we can resetAndTruncate() safely if the append fails - maybeReopenEarly(row.key); - RowIndexEntry index = writer.append(row); - if (!isOffline) + DecoratedKey key = partition.partitionKey(); + maybeReopenEarly(key); + RowIndexEntry index = writer.append(partition); + if (!isOffline && index != null) { - if (index == null) + boolean save = false; + for (SSTableReader reader : transaction.originals()) { - cfs.invalidateCachedRow(row.key); - } - else - { - boolean save = false; - for (SSTableReader reader : transaction.originals()) + if (reader.getCachedPosition(key, false) != null) { - if (reader.getCachedPosition(row.key, false) != null) - { - save = true; - break; - } + save = true; + break; } - if (save) - cachedKeys.put(row.key, index); } + if (save) + cachedKeys.put(key, index); } return index; } // attempts to append the row, if fails resets the writer position - public RowIndexEntry tryAppend(AbstractCompactedRow row) + public RowIndexEntry tryAppend(UnfilteredRowIterator partition) { writer.mark(); try { - return append(row); + return append(partition); } catch (Throwable t) { diff --git a/src/java/org/apache/cassandra/io/sstable/SSTableSimpleIterator.java b/src/java/org/apache/cassandra/io/sstable/SSTableSimpleIterator.java new file mode 100644 index 0000000000..1188de1ca7 --- /dev/null +++ b/src/java/org/apache/cassandra/io/sstable/SSTableSimpleIterator.java @@ -0,0 +1,176 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.cassandra.io.sstable; + +import java.io.DataInput; +import java.io.IOException; +import java.io.IOError; +import java.util.Iterator; + +import com.google.common.collect.AbstractIterator; + +import org.apache.cassandra.config.CFMetaData; +import org.apache.cassandra.config.ColumnDefinition; +import org.apache.cassandra.db.*; +import org.apache.cassandra.db.rows.*; +import org.apache.cassandra.io.util.FileDataInput; +import org.apache.cassandra.io.util.FileMark; +import org.apache.cassandra.net.MessagingService; + +/** + * Utility class to handle deserializing atom from sstables. + * + * Note that this is not a full fledged UnfilteredRowIterator. It's also not closeable, it is always + * the job of the user to close the underlying ressources. + */ +public abstract class SSTableSimpleIterator extends AbstractIterator implements Iterator +{ + protected final CFMetaData metadata; + protected final DataInput in; + protected final SerializationHelper helper; + + private SSTableSimpleIterator(CFMetaData metadata, DataInput in, SerializationHelper helper) + { + this.metadata = metadata; + this.in = in; + this.helper = helper; + } + + public static SSTableSimpleIterator create(CFMetaData metadata, DataInput in, SerializationHeader header, SerializationHelper helper, DeletionTime partitionDeletion) + { + if (helper.version < MessagingService.VERSION_30) + return new OldFormatIterator(metadata, in, helper, partitionDeletion); + else + return new CurrentFormatIterator(metadata, in, header, helper); + } + + public abstract Row readStaticRow() throws IOException; + + private static class CurrentFormatIterator extends SSTableSimpleIterator + { + private final SerializationHeader header; + + private final ReusableRow row; + private final RangeTombstoneMarker.Builder markerBuilder; + + private CurrentFormatIterator(CFMetaData metadata, DataInput in, SerializationHeader header, SerializationHelper helper) + { + super(metadata, in, helper); + this.header = header; + + int clusteringSize = metadata.comparator.size(); + Columns regularColumns = header == null ? metadata.partitionColumns().regulars : header.columns().regulars; + + this.row = new ReusableRow(clusteringSize, regularColumns, true, metadata.isCounter()); + this.markerBuilder = new RangeTombstoneMarker.Builder(clusteringSize); + } + + public Row readStaticRow() throws IOException + { + return header.hasStatic() + ? UnfilteredSerializer.serializer.deserializeStaticRow(in, header, helper) + : Rows.EMPTY_STATIC_ROW; + } + + protected Unfiltered computeNext() + { + try + { + Unfiltered.Kind kind = UnfilteredSerializer.serializer.deserialize(in, header, helper, row.writer(), markerBuilder.reset()); + + return kind == null + ? endOfData() + : (kind == Unfiltered.Kind.ROW ? row : markerBuilder.build()); + } + catch (IOException e) + { + throw new IOError(e); + } + } + } + + private static class OldFormatIterator extends SSTableSimpleIterator + { + private final UnfilteredDeserializer deserializer; + + private OldFormatIterator(CFMetaData metadata, DataInput in, SerializationHelper helper, DeletionTime partitionDeletion) + { + super(metadata, in, helper); + // We use an UnfilteredDeserializer because even though we don't need all it's fanciness, it happens to handle all + // the details we need for reading the old format. + this.deserializer = UnfilteredDeserializer.create(metadata, in, null, helper, partitionDeletion, false); + } + + public Row readStaticRow() throws IOException + { + if (metadata.isCompactTable()) + { + // For static compact tables, in the old format, static columns are intermingled with the other columns, so we + // need to extract them. Which imply 2 passes (one to extract the static, then one for other value). + if (metadata.isStaticCompactTable()) + { + // Because we don't support streaming from old file version, the only case we should get there is for compaction, + // where the DataInput should be a file based one. + assert in instanceof FileDataInput; + FileDataInput file = (FileDataInput)in; + FileMark mark = file.mark(); + Row staticRow = LegacyLayout.extractStaticColumns(metadata, file, metadata.partitionColumns().statics); + file.reset(mark); + + // We've extracted the static columns, so we must ignore them on the 2nd pass + ((UnfilteredDeserializer.OldFormatDeserializer)deserializer).setSkipStatic(); + return staticRow; + } + else + { + return Rows.EMPTY_STATIC_ROW; + } + } + + return deserializer.hasNext() && deserializer.nextIsStatic() + ? (Row)deserializer.readNext() + : Rows.EMPTY_STATIC_ROW; + + } + + protected Unfiltered computeNext() + { + try + { + if (!deserializer.hasNext()) + return endOfData(); + + Unfiltered unfiltered = deserializer.readNext(); + if (metadata.isStaticCompactTable() && unfiltered.kind() == Unfiltered.Kind.ROW) + { + Row row = (Row) unfiltered; + ColumnDefinition def = metadata.getColumnDefinition(LegacyLayout.encodeClustering(metadata, row.clustering())); + if (def != null && def.isStatic()) + return computeNext(); + } + return unfiltered; + } + catch (IOException e) + { + throw new IOError(e); + } + } + + } + +} diff --git a/src/java/org/apache/cassandra/io/sstable/SSTableSimpleUnsortedWriter.java b/src/java/org/apache/cassandra/io/sstable/SSTableSimpleUnsortedWriter.java index 4bb75bc4ee..a226585461 100644 --- a/src/java/org/apache/cassandra/io/sstable/SSTableSimpleUnsortedWriter.java +++ b/src/java/org/apache/cassandra/io/sstable/SSTableSimpleUnsortedWriter.java @@ -19,6 +19,7 @@ package org.apache.cassandra.io.sstable; import java.io.File; import java.io.IOException; +import java.nio.ByteBuffer; import java.util.Map; import java.util.TreeMap; import java.util.concurrent.BlockingQueue; @@ -28,30 +29,28 @@ import java.util.concurrent.TimeUnit; import com.google.common.base.Throwables; import org.apache.cassandra.config.CFMetaData; -import org.apache.cassandra.db.ArrayBackedSortedColumns; -import org.apache.cassandra.db.Cell; -import org.apache.cassandra.db.ColumnFamily; -import org.apache.cassandra.db.DecoratedKey; -import org.apache.cassandra.db.TypeSizes; -import org.apache.cassandra.db.marshal.AbstractType; +import org.apache.cassandra.config.ColumnDefinition; +import org.apache.cassandra.config.DatabaseDescriptor; +import org.apache.cassandra.db.*; +import org.apache.cassandra.db.rows.CellPath; +import org.apache.cassandra.db.partitions.PartitionUpdate; import org.apache.cassandra.dht.IPartitioner; -import org.apache.cassandra.io.compress.CompressionParameters; import org.apache.cassandra.io.sstable.format.SSTableWriter; +import org.apache.cassandra.utils.ByteBufferUtil; +import org.apache.cassandra.utils.FBUtilities; import org.apache.cassandra.utils.JVMStabilityInspector; /** * A SSTable writer that doesn't assume rows are in sorted order. + *

* This writer buffers rows in memory and then write them all in sorted order. * To avoid loading the entire data set in memory, the amount of rows buffered * is configurable. Each time the threshold is met, one SSTable will be * created (and the buffer be reseted). * - * @see AbstractSSTableSimpleWriter - * - * @deprecated this class is depracted in favor of {@link CQLSSTableWriter}. + * @see SSTableSimpleWriter */ -@Deprecated -public class SSTableSimpleUnsortedWriter extends AbstractSSTableSimpleWriter +class SSTableSimpleUnsortedWriter extends AbstractSSTableSimpleWriter { private static final Buffer SENTINEL = new Buffer(); @@ -62,92 +61,130 @@ public class SSTableSimpleUnsortedWriter extends AbstractSSTableSimpleWriter private final BlockingQueue writeQueue = new SynchronousQueue(); private final DiskWriter diskWriter = new DiskWriter(); - /** - * Create a new buffering writer. - * @param directory the directory where to write the sstables - * @param partitioner the partitioner - * @param keyspace the keyspace name - * @param columnFamily the column family name - * @param comparator the column family comparator - * @param subComparator the column family subComparator or null if not a Super column family. - * @param bufferSizeInMB the data size in MB before which a sstable is written and the buffer reseted. This correspond roughly to the written - * data size (i.e. the size of the create sstable). The actual size used in memory will be higher (by how much depends on the size of the - * columns you add). For 1GB of heap, a 128 bufferSizeInMB is probably a reasonable choice. If you experience OOM, this value should be lowered. - */ - public SSTableSimpleUnsortedWriter(File directory, - IPartitioner partitioner, - String keyspace, - String columnFamily, - AbstractType comparator, - AbstractType subComparator, - int bufferSizeInMB, - CompressionParameters compressParameters) + SSTableSimpleUnsortedWriter(File directory, CFMetaData metadata, IPartitioner partitioner, PartitionColumns columns, long bufferSizeInMB) { - this(directory, CFMetaData.denseCFMetaData(keyspace, columnFamily, comparator, subComparator).compressionParameters(compressParameters), partitioner, bufferSizeInMB); - } - - public SSTableSimpleUnsortedWriter(File directory, - IPartitioner partitioner, - String keyspace, - String columnFamily, - AbstractType comparator, - AbstractType subComparator, - int bufferSizeInMB) - { - this(directory, partitioner, keyspace, columnFamily, comparator, subComparator, bufferSizeInMB, new CompressionParameters(null)); - } - - public SSTableSimpleUnsortedWriter(File directory, CFMetaData metadata, IPartitioner partitioner, long bufferSizeInMB) - { - super(directory, metadata, partitioner); - bufferSize = bufferSizeInMB * 1024L * 1024L; + super(directory, metadata, partitioner, columns); + this.bufferSize = bufferSizeInMB * 1024L * 1024L; diskWriter.start(); } - protected void writeRow(DecoratedKey key, ColumnFamily columnFamily) throws IOException + PartitionUpdate getUpdateFor(DecoratedKey key) { - // Nothing to do since we'll sync if needed in addColumn. - } + assert key != null; - @Override - protected void addColumn(Cell cell) throws IOException - { - super.addColumn(cell); - countColumn(cell); - } - - protected void countColumn(Cell cell) throws IOException - { - currentSize += cell.serializedSize(metadata.comparator, TypeSizes.NATIVE); - - // We don't want to sync in writeRow() only as this might blow up the bufferSize for wide rows. - if (currentSize > bufferSize) - replaceColumnFamily(); - } - - protected ColumnFamily getColumnFamily() - { - ColumnFamily previous = buffer.get(currentKey); - // If the CF already exist in memory, we'll just continue adding to it + PartitionUpdate previous = buffer.get(key); if (previous == null) { - previous = createColumnFamily(); - buffer.put(currentKey, previous); - - // Since this new CF will be written by the next sync(), count its header. And a CF header - // on disk is: - // - the row key: 2 bytes size + key size bytes - // - the row level deletion infos: 4 + 8 bytes - currentSize += 14 + currentKey.getKey().remaining(); + previous = createPartitionUpdate(key); + count(PartitionUpdate.serializer.serializedSize(previous, formatType.info.getLatestVersion().correspondingMessagingVersion(), TypeSizes.NATIVE)); + previous.allowNewUpdates(); + buffer.put(key, previous); } return previous; } - protected ColumnFamily createColumnFamily() + private void count(long size) { - return ArrayBackedSortedColumns.factory.create(metadata); + currentSize += size; } + private void countCell(ColumnDefinition column, ByteBuffer value, LivenessInfo info, CellPath path) + { + // Note that the accounting of a cell is a bit inaccurate (it doesn't take some of the file format optimization into account) + // and the maintaining of the bufferSize is in general not perfect. This has always been the case for this class but we should + // improve that. In particular, what we count is closer to the serialized value, but it's debatable that it's the right thing + // to count since it will take a lot more space in memory and the bufferSize if first and foremost used to avoid OOM when + // using this writer. + + count(1); // Each cell has a byte flag on disk + + if (value.hasRemaining()) + count(column.type.writtenLength(value, TypeSizes.NATIVE)); + + count(8); // timestamp + if (info.hasLocalDeletionTime()) + count(4); + if (info.hasTTL()) + count(4); + + if (path != null) + { + assert path.size() == 1; + count(2 + path.get(0).remaining()); + } + } + + private void maybeSync() throws SyncException + { + try + { + if (currentSize > bufferSize) + sync(); + } + catch (IOException e) + { + // addColumn does not throw IOException but we want to report this to the user, + // so wrap it in a temporary RuntimeException that we'll catch in rawAddRow above. + throw new SyncException(e); + } + } + + private PartitionUpdate createPartitionUpdate(DecoratedKey key) + { + return new PartitionUpdate(metadata, key, columns, 4) + { + @Override + protected StaticWriter createStaticWriter() + { + return new StaticWriter() + { + @Override + public void writeCell(ColumnDefinition column, boolean isCounter, ByteBuffer value, LivenessInfo info, CellPath path) + { + super.writeCell(column, isCounter, value, info, path); + countCell(column, value, info, path); + } + + @Override + public void endOfRow() + { + super.endOfRow(); + maybeSync(); + } + }; + } + + @Override + protected Writer createWriter() + { + return new RegularWriter() + { + @Override + public void writeClusteringValue(ByteBuffer value) + { + super.writeClusteringValue(value); + count(2 + value.remaining()); + } + + @Override + public void writeCell(ColumnDefinition column, boolean isCounter, ByteBuffer value, LivenessInfo info, CellPath path) + { + super.writeCell(column, isCounter, value, info, path); + countCell(column, value, info, path); + } + + @Override + public void endOfRow() + { + super.endOfRow(); + maybeSync(); + } + }; + } + }; + } + + @Override public void close() throws IOException { sync(); @@ -163,23 +200,14 @@ public class SSTableSimpleUnsortedWriter extends AbstractSSTableSimpleWriter checkForWriterException(); } - // This is overridden by CQLSSTableWriter to hold off replacing column family until the next iteration through - protected void replaceColumnFamily() throws IOException - { - sync(); - } - protected void sync() throws IOException { if (buffer.isEmpty()) return; - columnFamily = null; put(buffer); buffer = new Buffer(); currentSize = 0; - columnFamily = getColumnFamily(); - buffer.setFirstInsertedKey(currentKey); } private void put(Buffer buffer) throws IOException @@ -211,56 +239,45 @@ public class SSTableSimpleUnsortedWriter extends AbstractSSTableSimpleWriter } } - // typedef - private static class Buffer extends TreeMap { - private DecoratedKey firstInsertedKey; - - public void setFirstInsertedKey(DecoratedKey firstInsertedKey) { - this.firstInsertedKey = firstInsertedKey; - } - - public DecoratedKey getFirstInsertedKey() { - return firstInsertedKey; + static class SyncException extends RuntimeException + { + SyncException(IOException ioe) + { + super(ioe); } } + //// typedef + static class Buffer extends TreeMap {} + private class DiskWriter extends Thread { volatile Throwable exception = null; public void run() { + while (true) { - while (true) + try { - try - { - Buffer b = writeQueue.take(); - if (b == SENTINEL) - return; + Buffer b = writeQueue.take(); + if (b == SENTINEL) + return; - try (SSTableWriter writer = getWriter();) - { - for (Map.Entry entry : b.entrySet()) - { - if (entry.getValue().getColumnCount() > 0) - writer.append(entry.getKey(), entry.getValue()); - else if (!entry.getKey().equals(b.getFirstInsertedKey())) - throw new AssertionError("Empty partition"); - } - - writer.finish(false); - } - } - catch (Throwable e) + try (SSTableWriter writer = createWriter()) { - JVMStabilityInspector.inspectThrowable(e); - // Keep only the first exception - if (exception == null) - exception = e; + for (Map.Entry entry : b.entrySet()) + writer.append(entry.getValue().unfilteredIterator()); + writer.finish(false); } } - + catch (Throwable e) + { + JVMStabilityInspector.inspectThrowable(e); + // Keep only the first exception + if (exception == null) + exception = e; + } } } } diff --git a/src/java/org/apache/cassandra/io/sstable/SSTableSimpleWriter.java b/src/java/org/apache/cassandra/io/sstable/SSTableSimpleWriter.java index 2601d6de9d..8a83242dbd 100644 --- a/src/java/org/apache/cassandra/io/sstable/SSTableSimpleWriter.java +++ b/src/java/org/apache/cassandra/io/sstable/SSTableSimpleWriter.java @@ -18,80 +18,93 @@ package org.apache.cassandra.io.sstable; import java.io.File; +import java.io.FilenameFilter; +import java.io.IOException; +import java.nio.ByteBuffer; +import java.util.HashSet; +import java.util.Set; +import java.util.concurrent.atomic.AtomicInteger; import com.google.common.base.Throwables; import org.apache.cassandra.config.CFMetaData; +import org.apache.cassandra.config.DatabaseDescriptor; import org.apache.cassandra.db.*; -import org.apache.cassandra.db.marshal.AbstractType; +import org.apache.cassandra.db.rows.RowStats; +import org.apache.cassandra.db.partitions.PartitionUpdate; import org.apache.cassandra.dht.IPartitioner; -import org.apache.cassandra.io.FSError; +import org.apache.cassandra.io.sstable.format.SSTableFormat; import org.apache.cassandra.io.sstable.format.SSTableWriter; +import org.apache.cassandra.service.ActiveRepairService; +import org.apache.cassandra.utils.CounterId; +import org.apache.cassandra.utils.FBUtilities; +import org.apache.cassandra.utils.Pair; /** * A SSTable writer that assumes rows are in (partitioner) sorted order. + *

* Contrarily to SSTableSimpleUnsortedWriter, this writer does not buffer * anything into memory, however it assumes that row are added in sorted order * (an exception will be thrown otherwise), which for the RandomPartitioner * means that rows should be added by increasing md5 of the row key. This is * rarely possible and SSTableSimpleUnsortedWriter should most of the time be * prefered. - * - * @see AbstractSSTableSimpleWriter - * - * @deprecated this class is depracted in favor of {@link CQLSSTableWriter}. */ -@Deprecated -public class SSTableSimpleWriter extends AbstractSSTableSimpleWriter +class SSTableSimpleWriter extends AbstractSSTableSimpleWriter { - private final SSTableWriter writer; + protected DecoratedKey currentKey; + protected PartitionUpdate update; - /** - * Create a new writer. - * @param directory the directory where to write the sstable - * @param partitioner the partitioner - * @param keyspace the keyspace name - * @param columnFamily the column family name - * @param comparator the column family comparator - * @param subComparator the column family subComparator or null if not a Super column family. - */ - public SSTableSimpleWriter(File directory, - IPartitioner partitioner, - String keyspace, - String columnFamily, - AbstractType comparator, - AbstractType subComparator) + private SSTableWriter writer; + + protected SSTableSimpleWriter(File directory, CFMetaData metadata, IPartitioner partitioner, PartitionColumns columns) { - this(directory, CFMetaData.denseCFMetaData(keyspace, columnFamily, comparator, subComparator), partitioner); + super(directory, metadata, partitioner, columns); } - public SSTableSimpleWriter(File directory, CFMetaData metadata, IPartitioner partitioner) + private SSTableWriter getOrCreateWriter() { - super(directory, metadata, partitioner); - writer = getWriter(); + if (writer == null) + writer = createWriter(); + + return writer; + } + + PartitionUpdate getUpdateFor(DecoratedKey key) throws IOException + { + assert key != null; + + // If that's not the current key, write the current one if necessary and create a new + // update for the new key. + if (!key.equals(currentKey)) + { + if (update != null) + writePartition(update); + currentKey = key; + update = new PartitionUpdate(metadata, currentKey, columns, 4); + } + + assert update != null; + return update; } public void close() { try { - if (currentKey != null) - writeRow(currentKey, columnFamily); - writer.finish(false); + if (update != null) + writePartition(update); + if (writer != null) + writer.finish(false); } catch (Throwable t) { - throw Throwables.propagate(writer.abort(t)); + throw Throwables.propagate(writer == null ? t : writer.abort(t)); } } - protected void writeRow(DecoratedKey key, ColumnFamily columnFamily) + private void writePartition(PartitionUpdate update) throws IOException { - writer.append(key, columnFamily); - } - - protected ColumnFamily getColumnFamily() - { - return ArrayBackedSortedColumns.factory.create(metadata); + getOrCreateWriter().append(update.unfilteredIterator()); } } diff --git a/src/java/org/apache/cassandra/io/sstable/format/SSTableFormat.java b/src/java/org/apache/cassandra/io/sstable/format/SSTableFormat.java index ca003b64b4..1286f16f43 100644 --- a/src/java/org/apache/cassandra/io/sstable/format/SSTableFormat.java +++ b/src/java/org/apache/cassandra/io/sstable/format/SSTableFormat.java @@ -20,11 +20,9 @@ package org.apache.cassandra.io.sstable.format; import com.google.common.base.CharMatcher; import com.google.common.collect.ImmutableList; import org.apache.cassandra.config.CFMetaData; -import org.apache.cassandra.db.ColumnSerializer; -import org.apache.cassandra.db.OnDiskAtom; +import org.apache.cassandra.db.LegacyLayout; import org.apache.cassandra.db.RowIndexEntry; -import org.apache.cassandra.db.columniterator.OnDiskAtomIterator; -import org.apache.cassandra.db.compaction.AbstractCompactedRow; +import org.apache.cassandra.db.SerializationHeader; import org.apache.cassandra.db.compaction.CompactionController; import org.apache.cassandra.io.sstable.format.big.BigFormat; import org.apache.cassandra.io.util.FileDataInput; @@ -45,11 +43,7 @@ public interface SSTableFormat SSTableWriter.Factory getWriterFactory(); SSTableReader.Factory getReaderFactory(); - Iterator getOnDiskIterator(FileDataInput in, ColumnSerializer.Flag flag, int expireBefore, CFMetaData cfm, Version version); - - AbstractCompactedRow getCompactedRowWriter(CompactionController controller, ImmutableList onDiskAtomIterators); - - RowIndexEntry.IndexSerializer getIndexSerializer(CFMetaData cfm); + RowIndexEntry.IndexSerializer getIndexSerializer(CFMetaData cfm, Version version, SerializationHeader header); public static enum Type { @@ -62,6 +56,7 @@ public interface SSTableFormat public final SSTableFormat info; public final String name; + private Type(String name, SSTableFormat info) { //Since format comes right after generation diff --git a/src/java/org/apache/cassandra/io/sstable/format/SSTableReader.java b/src/java/org/apache/cassandra/io/sstable/format/SSTableReader.java index 1458461de1..a38e60ff48 100644 --- a/src/java/org/apache/cassandra/io/sstable/format/SSTableReader.java +++ b/src/java/org/apache/cassandra/io/sstable/format/SSTableReader.java @@ -41,10 +41,9 @@ import org.apache.cassandra.concurrent.DebuggableThreadPoolExecutor; import org.apache.cassandra.concurrent.ScheduledExecutors; import org.apache.cassandra.config.*; import org.apache.cassandra.db.*; -import org.apache.cassandra.db.columniterator.OnDiskAtomIterator; +import org.apache.cassandra.db.filter.ColumnFilter; +import org.apache.cassandra.db.rows.*; import org.apache.cassandra.db.commitlog.ReplayPosition; -import org.apache.cassandra.db.composites.CellName; -import org.apache.cassandra.db.filter.ColumnSlice; import org.apache.cassandra.db.index.SecondaryIndex; import org.apache.cassandra.db.lifecycle.Tracker; import org.apache.cassandra.dht.*; @@ -199,6 +198,8 @@ public abstract class SSTableReader extends SSTable implements SelfRefCounted sstableMetadata = descriptor.getMetadataSerializer().deserialize(descriptor, - EnumSet.of(MetadataType.VALIDATION, MetadataType.STATS)); + EnumSet types = EnumSet.of(MetadataType.VALIDATION, MetadataType.STATS, MetadataType.HEADER); + Map sstableMetadata = descriptor.getMetadataSerializer().deserialize(descriptor, types); + ValidationMetadata validationMetadata = (ValidationMetadata) sstableMetadata.get(MetadataType.VALIDATION); StatsMetadata statsMetadata = (StatsMetadata) sstableMetadata.get(MetadataType.STATS); + SerializationHeader.Component header = (SerializationHeader.Component) sstableMetadata.get(MetadataType.HEADER); // Check if sstable is created using same partitioner. // Partitioner can be null, which indicates older version of sstable or no stats available. @@ -392,8 +395,14 @@ public abstract class SSTableReader extends SSTable implements SelfRefCounted sstableMetadata = descriptor.getMetadataSerializer().deserialize(descriptor, - EnumSet.of(MetadataType.VALIDATION, MetadataType.STATS)); + // For the 3.0+ sstable format, the (misnomed) stats component hold the serialization header which we need to deserialize the sstable content + assert !descriptor.version.storeRows() || components.contains(Component.STATS) : "Stats component is missing for sstable " + descriptor; + + EnumSet types = EnumSet.of(MetadataType.VALIDATION, MetadataType.STATS, MetadataType.HEADER); + Map sstableMetadata = descriptor.getMetadataSerializer().deserialize(descriptor, types); ValidationMetadata validationMetadata = (ValidationMetadata) sstableMetadata.get(MetadataType.VALIDATION); StatsMetadata statsMetadata = (StatsMetadata) sstableMetadata.get(MetadataType.STATS); + SerializationHeader.Component header = (SerializationHeader.Component) sstableMetadata.get(MetadataType.HEADER); + assert !descriptor.version.storeRows() || header != null; // Check if sstable is created using same partitioner. // Partitioner can be null, which indicates older version of sstable or no stats available. @@ -438,8 +452,14 @@ public abstract class SSTableReader extends SSTable implements SelfRefCounted sstables) @@ -736,12 +760,12 @@ public abstract class SSTableReader extends SSTable implements SelfRefCounted= 0 */ - public long getIndexScanPosition(RowPosition key) + public long getIndexScanPosition(PartitionPosition key) { if (openReason == OpenReason.MOVED_START && key.compareTo(first) < 0) key = first; @@ -1287,8 +1312,8 @@ public abstract class SSTableReader extends SSTable implements SelfRefCounted range : Range.normalize(ranges)) { - RowPosition leftPosition = range.left.maxKeyBound(); - RowPosition rightPosition = range.right.maxKeyBound(); + PartitionPosition leftPosition = range.left.maxKeyBound(); + PartitionPosition rightPosition = range.right.maxKeyBound(); int left = summary.binarySearch(leftPosition); if (left < 0) @@ -1382,9 +1407,9 @@ public abstract class SSTableReader extends SSTable implements SelfRefCounted bounds = Range.makeRowRange(range); - RowPosition leftBound = bounds.left.compareTo(first) > 0 ? bounds.left : first.getToken().minKeyBound(); - RowPosition rightBound = bounds.right.isMinimum() ? last.getToken().maxKeyBound() : bounds.right; + AbstractBounds bounds = Range.makeRowRange(range); + PartitionPosition leftBound = bounds.left.compareTo(first) > 0 ? bounds.left : first.getToken().minKeyBound(); + PartitionPosition rightBound = bounds.right.isMinimum() ? last.getToken().maxKeyBound() : bounds.right; if (leftBound.compareTo(last) > 0 || rightBound.compareTo(first) < 0) continue; @@ -1455,14 +1480,14 @@ public abstract class SSTableReader extends SSTable implements SelfRefCounted columns); - public abstract OnDiskAtomIterator iterator(FileDataInput file, DecoratedKey key, SortedSet columns, RowIndexEntry indexEntry); - - //Corresponds to a slice query - public abstract OnDiskAtomIterator iterator(DecoratedKey key, ColumnSlice[] slices, boolean reverse); - public abstract OnDiskAtomIterator iterator(FileDataInput file, DecoratedKey key, ColumnSlice[] slices, boolean reversed, RowIndexEntry indexEntry); + public abstract SliceableUnfilteredRowIterator iterator(DecoratedKey key, ColumnFilter selectedColumns, boolean reversed, boolean isForThrift); + public abstract SliceableUnfilteredRowIterator iterator(FileDataInput file, DecoratedKey key, RowIndexEntry indexEntry, ColumnFilter selectedColumns, boolean reversed, boolean isForThrift); /** * Finds and returns the first key beyond a given token in this SSTable or null if no such key exists. */ - public DecoratedKey firstKeyBeyond(RowPosition token) + public DecoratedKey firstKeyBeyond(PartitionPosition token) { if (token.compareTo(first) < 0) return first; @@ -1589,19 +1609,14 @@ public abstract class SSTableReader extends SSTable implements SelfRefCounted> ranges, RateLimiter limiter); /** - * + * @param columns the columns to return. * @param dataRange filter to use when reading the columns * @return A Scanner for seeking over the rows of the SSTable. */ - public abstract ISSTableScanner getScanner(DataRange dataRange, RateLimiter limiter); + public abstract ISSTableScanner getScanner(ColumnFilter columns, DataRange dataRange, RateLimiter limiter, boolean isForThrift); @@ -1761,6 +1783,43 @@ public abstract class SSTableReader extends SSTable implements SelfRefCounted getAncestors() { try @@ -1850,6 +1909,34 @@ public abstract class SSTableReader extends SSTable implements SelfRefCounted 0); + } + + private int compare(List values1, List values2) + { + ClusteringComparator comparator = metadata.comparator; + for (int i = 0; i < Math.min(values1.size(), values2.size()); i++) + { + int cmp = comparator.subtype(i).compare(values1.get(i), values2.get(i)); + if (cmp != 0) + return cmp; + } + return 0; + } + public static class SizeComparator implements Comparator { public int compare(SSTableReader o1, SSTableReader o2) @@ -2172,7 +2259,8 @@ public abstract class SSTableReader extends SSTable implements SelfRefCounted components(CFMetaData metadata) @@ -141,19 +148,18 @@ public abstract class SSTableWriter extends SSTable implements Transactional return components; } - public abstract void mark(); - /** - * @param row - * @return null if the row was compacted away entirely; otherwise, the PK index entry for this row + * Appends partition data to this writer. + * + * @param iterator the partition to write + * @return the created index entry if something was written, that is if {@code iterator} + * wasn't empty, {@code null} otherwise. + * + * @throws FSWriteError if a write to the dataFile fails */ - public abstract RowIndexEntry append(AbstractCompactedRow row); - - public abstract void append(DecoratedKey decoratedKey, ColumnFamily cf); - - public abstract long appendFromStream(DecoratedKey key, CFMetaData metadata, DataInput in, Version version) throws IOException; + public abstract RowIndexEntry append(UnfilteredRowIterator iterator); public abstract long getFilePointer(); @@ -244,7 +250,9 @@ public abstract class SSTableWriter extends SSTable implements Transactional protected Map finalizeMetadata() { return metadataCollector.finalizeMetadata(partitioner.getClass().getCanonicalName(), - metadata.getBloomFilterFpChance(), repairedAt); + metadata.getBloomFilterFpChance(), + repairedAt, + header); } protected StatsMetadata statsMetadata() @@ -276,6 +284,12 @@ public abstract class SSTableWriter extends SSTable implements Transactional public static abstract class Factory { - public abstract SSTableWriter open(Descriptor descriptor, long keyCount, long repairedAt, CFMetaData metadata, IPartitioner partitioner, MetadataCollector metadataCollector); + public abstract SSTableWriter open(Descriptor descriptor, + long keyCount, + long repairedAt, + CFMetaData metadata, + IPartitioner partitioner, + MetadataCollector metadataCollector, + SerializationHeader header); } } diff --git a/src/java/org/apache/cassandra/io/sstable/format/Version.java b/src/java/org/apache/cassandra/io/sstable/format/Version.java index faaa89e210..8077a459c5 100644 --- a/src/java/org/apache/cassandra/io/sstable/format/Version.java +++ b/src/java/org/apache/cassandra/io/sstable/format/Version.java @@ -52,6 +52,10 @@ public abstract class Version public abstract boolean hasNewFileName(); + public abstract boolean storeRows(); + + public abstract int correspondingMessagingVersion(); // Only use by storage that 'storeRows' so far + public String getVersion() { return version; @@ -73,6 +77,7 @@ public abstract class Version } abstract public boolean isCompatible(); + abstract public boolean isCompatibleForStreaming(); @Override public String toString() diff --git a/src/java/org/apache/cassandra/io/sstable/format/big/BigFormat.java b/src/java/org/apache/cassandra/io/sstable/format/big/BigFormat.java index a1e32cf977..fd0b5d5cf3 100644 --- a/src/java/org/apache/cassandra/io/sstable/format/big/BigFormat.java +++ b/src/java/org/apache/cassandra/io/sstable/format/big/BigFormat.java @@ -17,16 +17,14 @@ */ package org.apache.cassandra.io.sstable.format.big; +import java.util.Iterator; +import java.util.Set; + import com.google.common.collect.ImmutableList; import org.apache.cassandra.config.CFMetaData; -import org.apache.cassandra.db.AbstractCell; -import org.apache.cassandra.db.ColumnSerializer; -import org.apache.cassandra.db.OnDiskAtom; import org.apache.cassandra.db.RowIndexEntry; -import org.apache.cassandra.db.columniterator.OnDiskAtomIterator; -import org.apache.cassandra.db.compaction.AbstractCompactedRow; +import org.apache.cassandra.db.SerializationHeader; import org.apache.cassandra.db.compaction.CompactionController; -import org.apache.cassandra.db.compaction.LazilyCompactedRow; import org.apache.cassandra.dht.IPartitioner; import org.apache.cassandra.io.sstable.Component; import org.apache.cassandra.io.sstable.Descriptor; @@ -38,9 +36,7 @@ import org.apache.cassandra.io.sstable.format.Version; import org.apache.cassandra.io.sstable.metadata.MetadataCollector; import org.apache.cassandra.io.sstable.metadata.StatsMetadata; import org.apache.cassandra.io.util.FileDataInput; - -import java.util.Iterator; -import java.util.Set; +import org.apache.cassandra.net.MessagingService; /** * Legacy bigtable format @@ -82,38 +78,26 @@ public class BigFormat implements SSTableFormat } @Override - public Iterator getOnDiskIterator(FileDataInput in, ColumnSerializer.Flag flag, int expireBefore, CFMetaData cfm, Version version) + public RowIndexEntry.IndexSerializer getIndexSerializer(CFMetaData metadata, Version version, SerializationHeader header) { - return AbstractCell.onDiskIterator(in, flag, expireBefore, version, cfm.comparator); - } - - @Override - public AbstractCompactedRow getCompactedRowWriter(CompactionController controller, ImmutableList onDiskAtomIterators) - { - return new LazilyCompactedRow(controller, onDiskAtomIterators); - } - - @Override - public RowIndexEntry.IndexSerializer getIndexSerializer(CFMetaData cfMetaData) - { - return new RowIndexEntry.Serializer(new IndexHelper.IndexInfo.Serializer(cfMetaData.comparator)); + return new RowIndexEntry.Serializer(metadata, version, header); } static class WriterFactory extends SSTableWriter.Factory { @Override - public SSTableWriter open(Descriptor descriptor, long keyCount, long repairedAt, CFMetaData metadata, IPartitioner partitioner, MetadataCollector metadataCollector) + public SSTableWriter open(Descriptor descriptor, long keyCount, long repairedAt, CFMetaData metadata, IPartitioner partitioner, MetadataCollector metadataCollector, SerializationHeader header) { - return new BigTableWriter(descriptor, keyCount, repairedAt, metadata, partitioner, metadataCollector); + return new BigTableWriter(descriptor, keyCount, repairedAt, metadata, partitioner, metadataCollector, header); } } static class ReaderFactory extends SSTableReader.Factory { @Override - public SSTableReader open(Descriptor descriptor, Set components, CFMetaData metadata, IPartitioner partitioner, Long maxDataAge, StatsMetadata sstableMetadata, SSTableReader.OpenReason openReason) + public SSTableReader open(Descriptor descriptor, Set components, CFMetaData metadata, IPartitioner partitioner, Long maxDataAge, StatsMetadata sstableMetadata, SSTableReader.OpenReason openReason, SerializationHeader header) { - return new BigTableReader(descriptor, components, metadata, partitioner, maxDataAge, sstableMetadata, openReason); + return new BigTableReader(descriptor, components, metadata, partitioner, maxDataAge, sstableMetadata, openReason, header); } } @@ -143,6 +127,8 @@ public class BigFormat implements SSTableFormat private final boolean hasRepairedAt; private final boolean tracksLegacyCounterShards; private final boolean newFileName; + public final boolean storeRows; + public final int correspondingMessagingVersion; // Only use by storage that 'storeRows' so far public BigVersion(String version) { @@ -155,6 +141,8 @@ public class BigFormat implements SSTableFormat hasRepairedAt = version.compareTo("ka") >= 0; tracksLegacyCounterShards = version.compareTo("ka") >= 0; newFileName = version.compareTo("la") >= 0; + storeRows = version.compareTo("la") >= 0; + correspondingMessagingVersion = storeRows ? MessagingService.VERSION_30 : MessagingService.VERSION_21; } @Override @@ -199,10 +187,28 @@ public class BigFormat implements SSTableFormat return newFileName; } + @Override + public boolean storeRows() + { + return storeRows; + } + + @Override + public int correspondingMessagingVersion() + { + return correspondingMessagingVersion; + } + @Override public boolean isCompatible() { return version.compareTo(earliest_supported_version) >= 0 && version.charAt(0) <= current_version.charAt(0); } + + @Override + public boolean isCompatibleForStreaming() + { + return isCompatible() && version.charAt(0) == current_version.charAt(0); + } } } diff --git a/src/java/org/apache/cassandra/io/sstable/format/big/BigTableReader.java b/src/java/org/apache/cassandra/io/sstable/format/big/BigTableReader.java index 3f375e7c0c..7a7b91326d 100644 --- a/src/java/org/apache/cassandra/io/sstable/format/big/BigTableReader.java +++ b/src/java/org/apache/cassandra/io/sstable/format/big/BigTableReader.java @@ -20,13 +20,11 @@ package org.apache.cassandra.io.sstable.format.big; import com.google.common.util.concurrent.RateLimiter; import org.apache.cassandra.cache.KeyCacheKey; import org.apache.cassandra.config.CFMetaData; -import org.apache.cassandra.db.DataRange; -import org.apache.cassandra.db.DecoratedKey; -import org.apache.cassandra.db.RowIndexEntry; -import org.apache.cassandra.db.RowPosition; -import org.apache.cassandra.db.columniterator.OnDiskAtomIterator; -import org.apache.cassandra.db.composites.CellName; -import org.apache.cassandra.db.filter.ColumnSlice; +import org.apache.cassandra.db.*; +import org.apache.cassandra.db.rows.SliceableUnfilteredRowIterator; +import org.apache.cassandra.db.filter.ColumnFilter; +import org.apache.cassandra.db.columniterator.SSTableIterator; +import org.apache.cassandra.db.columniterator.SSTableReversedIterator; import org.apache.cassandra.dht.IPartitioner; import org.apache.cassandra.dht.Range; import org.apache.cassandra.dht.Token; @@ -37,7 +35,6 @@ import org.apache.cassandra.io.sstable.ISSTableScanner; import org.apache.cassandra.io.sstable.format.SSTableReader; import org.apache.cassandra.io.sstable.metadata.StatsMetadata; import org.apache.cassandra.io.util.FileDataInput; -import org.apache.cassandra.io.util.FileUtils; import org.apache.cassandra.tracing.Tracing; import org.apache.cassandra.utils.ByteBufferUtil; import org.slf4j.Logger; @@ -55,40 +52,44 @@ public class BigTableReader extends SSTableReader { private static final Logger logger = LoggerFactory.getLogger(BigTableReader.class); - BigTableReader(Descriptor desc, Set components, CFMetaData metadata, IPartitioner partitioner, Long maxDataAge, StatsMetadata sstableMetadata, OpenReason openReason) + BigTableReader(Descriptor desc, Set components, CFMetaData metadata, IPartitioner partitioner, Long maxDataAge, StatsMetadata sstableMetadata, OpenReason openReason, SerializationHeader header) { - super(desc, components, metadata, partitioner, maxDataAge, sstableMetadata, openReason); + super(desc, components, metadata, partitioner, maxDataAge, sstableMetadata, openReason, header); } - public OnDiskAtomIterator iterator(DecoratedKey key, SortedSet columns) + public SliceableUnfilteredRowIterator iterator(DecoratedKey key, ColumnFilter selectedColumns, boolean reversed, boolean isForThrift) { - return new SSTableNamesIterator(this, key, columns); + return reversed + ? new SSTableReversedIterator(this, key, selectedColumns, isForThrift) + : new SSTableIterator(this, key, selectedColumns, isForThrift); } - public OnDiskAtomIterator iterator(FileDataInput input, DecoratedKey key, SortedSet columns, RowIndexEntry indexEntry ) + public SliceableUnfilteredRowIterator iterator(FileDataInput file, DecoratedKey key, RowIndexEntry indexEntry, ColumnFilter selectedColumns, boolean reversed, boolean isForThrift) { - return new SSTableNamesIterator(this, input, key, columns, indexEntry); + return reversed + ? new SSTableReversedIterator(this, file, key, indexEntry, selectedColumns, isForThrift) + : new SSTableIterator(this, file, key, indexEntry, selectedColumns, isForThrift); } - public OnDiskAtomIterator iterator(DecoratedKey key, ColumnSlice[] slices, boolean reverse) - { - return new SSTableSliceIterator(this, key, slices, reverse); - } - - public OnDiskAtomIterator iterator(FileDataInput input, DecoratedKey key, ColumnSlice[] slices, boolean reverse, RowIndexEntry indexEntry) - { - return new SSTableSliceIterator(this, input, key, slices, reverse, indexEntry); - } /** - * + * @param columns the columns to return. * @param dataRange filter to use when reading the columns * @return A Scanner for seeking over the rows of the SSTable. */ - public ISSTableScanner getScanner(DataRange dataRange, RateLimiter limiter) + public ISSTableScanner getScanner(ColumnFilter columns, DataRange dataRange, RateLimiter limiter, boolean isForThrift) { - return BigTableScanner.getScanner(this, dataRange, limiter); + return BigTableScanner.getScanner(this, columns, dataRange, limiter, isForThrift); } + /** + * Direct I/O SSTableScanner over the full sstable. + * + * @return A Scanner for reading the full SSTable. + */ + public ISSTableScanner getScanner(RateLimiter limiter) + { + return BigTableScanner.getScanner(this, limiter); + } /** * Direct I/O SSTableScanner over a defined collection of ranges of tokens. @@ -109,7 +110,7 @@ public class BigTableReader extends SSTableReader * @param updateCacheAndStats true if updating stats and cache * @return The index entry corresponding to the key, or null if the key is not present */ - protected RowIndexEntry getPosition(RowPosition key, Operator op, boolean updateCacheAndStats, boolean permitMatchPastLast) + protected RowIndexEntry getPosition(PartitionPosition key, Operator op, boolean updateCacheAndStats, boolean permitMatchPastLast) { if (op == Operator.EQ) { @@ -212,7 +213,7 @@ public class BigTableReader extends SSTableReader if (opSatisfied) { // read data position from index entry - RowIndexEntry indexEntry = rowIndexEntrySerializer.deserialize(in, descriptor.version); + RowIndexEntry indexEntry = rowIndexEntrySerializer.deserialize(in); if (exactMatch && updateCacheAndStats) { assert key instanceof DecoratedKey; // key can be == to the index key only if it's a true row key diff --git a/src/java/org/apache/cassandra/io/sstable/format/big/BigTableScanner.java b/src/java/org/apache/cassandra/io/sstable/format/big/BigTableScanner.java index d477152a5b..0794e90db9 100644 --- a/src/java/org/apache/cassandra/io/sstable/format/big/BigTableScanner.java +++ b/src/java/org/apache/cassandra/io/sstable/format/big/BigTableScanner.java @@ -22,16 +22,13 @@ import java.util.*; import java.util.concurrent.atomic.AtomicBoolean; import com.google.common.collect.AbstractIterator; -import com.google.common.collect.Ordering; +import com.google.common.collect.Iterators; import com.google.common.util.concurrent.RateLimiter; -import org.apache.cassandra.db.DataRange; -import org.apache.cassandra.db.DecoratedKey; -import org.apache.cassandra.db.RowIndexEntry; -import org.apache.cassandra.db.RowPosition; -import org.apache.cassandra.db.columniterator.IColumnIteratorFactory; -import org.apache.cassandra.db.columniterator.LazyColumnIterator; -import org.apache.cassandra.db.columniterator.OnDiskAtomIterator; +import org.apache.cassandra.db.*; +import org.apache.cassandra.db.rows.*; +import org.apache.cassandra.db.filter.*; +import org.apache.cassandra.db.partitions.*; import org.apache.cassandra.dht.AbstractBounds; import org.apache.cassandra.dht.AbstractBounds.Boundary; import org.apache.cassandra.dht.Bounds; @@ -57,18 +54,27 @@ public class BigTableScanner implements ISSTableScanner protected final RandomAccessReader ifile; public final SSTableReader sstable; - private final Iterator> rangeIterator; - private AbstractBounds currentRange; + private final Iterator> rangeIterator; + private AbstractBounds currentRange; + private final ColumnFilter columns; private final DataRange dataRange; private final RowIndexEntry.IndexSerializer rowIndexEntrySerializer; + private final boolean isForThrift; - protected Iterator iterator; + protected UnfilteredPartitionIterator iterator; - public static ISSTableScanner getScanner(SSTableReader sstable, DataRange dataRange, RateLimiter limiter) + // Full scan of the sstables + public static ISSTableScanner getScanner(SSTableReader sstable, RateLimiter limiter) { - return new BigTableScanner(sstable, dataRange, limiter); + return new BigTableScanner(sstable, ColumnFilter.all(sstable.metadata), null, limiter, false, Iterators.singletonIterator(fullRange(sstable))); } + + public static ISSTableScanner getScanner(SSTableReader sstable, ColumnFilter columns, DataRange dataRange, RateLimiter limiter, boolean isForThrift) + { + return new BigTableScanner(sstable, columns, dataRange, limiter, isForThrift, makeBounds(sstable, dataRange).iterator()); + } + public static ISSTableScanner getScanner(SSTableReader sstable, Collection> tokenRanges, RateLimiter limiter) { // We want to avoid allocating a SSTableScanner if the range don't overlap the sstable (#5249) @@ -76,60 +82,54 @@ public class BigTableScanner implements ISSTableScanner if (positions.isEmpty()) return new EmptySSTableScanner(sstable.getFilename()); - return new BigTableScanner(sstable, tokenRanges, limiter); + return new BigTableScanner(sstable, ColumnFilter.all(sstable.metadata), null, limiter, false, makeBounds(sstable, tokenRanges).iterator()); } - /** - * @param sstable SSTable to scan; must not be null - * @param dataRange a single range to scan; must not be null - * @param limiter background i/o RateLimiter; may be null - */ - private BigTableScanner(SSTableReader sstable, DataRange dataRange, RateLimiter limiter) + private BigTableScanner(SSTableReader sstable, ColumnFilter columns, DataRange dataRange, RateLimiter limiter, boolean isForThrift, Iterator> rangeIterator) { assert sstable != null; this.dfile = limiter == null ? sstable.openDataReader() : sstable.openDataReader(limiter); this.ifile = sstable.openIndexReader(); this.sstable = sstable; + this.columns = columns; this.dataRange = dataRange; - this.rowIndexEntrySerializer = sstable.descriptor.version.getSSTableFormat().getIndexSerializer(sstable.metadata); - - List> boundsList = new ArrayList<>(2); - addRange(dataRange.keyRange(), boundsList); - this.rangeIterator = boundsList.iterator(); + this.rowIndexEntrySerializer = sstable.descriptor.version.getSSTableFormat().getIndexSerializer(sstable.metadata, + sstable.descriptor.version, + sstable.header); + this.isForThrift = isForThrift; + this.rangeIterator = rangeIterator; } - /** - * @param sstable SSTable to scan; must not be null - * @param tokenRanges A set of token ranges to scan - * @param limiter background i/o RateLimiter; may be null - */ - private BigTableScanner(SSTableReader sstable, Collection> tokenRanges, RateLimiter limiter) + private static List> makeBounds(SSTableReader sstable, Collection> tokenRanges) { - assert sstable != null; - - this.dfile = limiter == null ? sstable.openDataReader() : sstable.openDataReader(limiter); - this.ifile = sstable.openIndexReader(); - this.sstable = sstable; - this.dataRange = null; - this.rowIndexEntrySerializer = sstable.descriptor.version.getSSTableFormat().getIndexSerializer(sstable.metadata); - - List> boundsList = new ArrayList<>(tokenRanges.size()); + List> boundsList = new ArrayList<>(tokenRanges.size()); for (Range range : Range.normalize(tokenRanges)) - addRange(Range.makeRowRange(range), boundsList); - - this.rangeIterator = boundsList.iterator(); + addRange(sstable, Range.makeRowRange(range), boundsList); + return boundsList; } - private void addRange(AbstractBounds requested, List> boundsList) + private static List> makeBounds(SSTableReader sstable, DataRange dataRange) + { + List> boundsList = new ArrayList<>(2); + addRange(sstable, dataRange.keyRange(), boundsList); + return boundsList; + } + + private static AbstractBounds fullRange(SSTableReader sstable) + { + return new Bounds(sstable.first, sstable.last); + } + + private static void addRange(SSTableReader sstable, AbstractBounds requested, List> boundsList) { if (requested instanceof Range && ((Range)requested).isWrapAround()) { if (requested.right.compareTo(sstable.first) >= 0) { // since we wrap, we must contain the whole sstable prior to stopKey() - Boundary left = new Boundary(sstable.first, true); - Boundary right; + Boundary left = new Boundary(sstable.first, true); + Boundary right; right = requested.rightBoundary(); right = minRight(right, sstable.last, true); if (!isEmpty(left, right)) @@ -138,8 +138,8 @@ public class BigTableScanner implements ISSTableScanner if (requested.left.compareTo(sstable.last) <= 0) { // since we wrap, we must contain the whole sstable after dataRange.startKey() - Boundary right = new Boundary(sstable.last, true); - Boundary left; + Boundary right = new Boundary(sstable.last, true); + Boundary left; left = requested.leftBoundary(); left = maxLeft(left, sstable.first, true); if (!isEmpty(left, right)) @@ -149,12 +149,12 @@ public class BigTableScanner implements ISSTableScanner else { assert requested.left.compareTo(requested.right) <= 0 || requested.right.isMinimum(); - Boundary left, right; + Boundary left, right; left = requested.leftBoundary(); right = requested.rightBoundary(); left = maxLeft(left, sstable.first, true); // apparently isWrapAround() doesn't count Bounds that extend to the limit (min) as wrapping - right = requested.right.isMinimum() ? new Boundary(sstable.last, true) + right = requested.right.isMinimum() ? new Boundary(sstable.last, true) : minRight(right, sstable.last, true); if (!isEmpty(left, right)) boundsList.add(AbstractBounds.bounds(left, right)); @@ -193,10 +193,18 @@ public class BigTableScanner implements ISSTableScanner } } - public void close() throws IOException + public void close() { - if (isClosed.compareAndSet(false, true)) - FileUtils.close(dfile, ifile); + try + { + if (isClosed.compareAndSet(false, true)) + FileUtils.close(dfile, ifile); + } + catch (IOException e) + { + sstable.markSuspect(); + throw new CorruptSSTableException(e, sstable.getFilename()); + } } public long getLengthInBytes() @@ -214,6 +222,11 @@ public class BigTableScanner implements ISSTableScanner return sstable.toString(); } + public boolean isForThrift() + { + return isForThrift; + } + public boolean hasNext() { if (iterator == null) @@ -221,7 +234,7 @@ public class BigTableScanner implements ISSTableScanner return iterator.hasNext(); } - public OnDiskAtomIterator next() + public UnfilteredRowIterator next() { if (iterator == null) iterator = createIterator(); @@ -233,19 +246,24 @@ public class BigTableScanner implements ISSTableScanner throw new UnsupportedOperationException(); } - private Iterator createIterator() + private UnfilteredPartitionIterator createIterator() { return new KeyScanningIterator(); } - protected class KeyScanningIterator extends AbstractIterator + protected class KeyScanningIterator extends AbstractIterator implements UnfilteredPartitionIterator { private DecoratedKey nextKey; private RowIndexEntry nextEntry; private DecoratedKey currentKey; private RowIndexEntry currentEntry; - protected OnDiskAtomIterator computeNext() + public boolean isForThrift() + { + return isForThrift; + } + + protected UnfilteredRowIterator computeNext() { try { @@ -264,7 +282,7 @@ public class BigTableScanner implements ISSTableScanner return endOfData(); currentKey = sstable.partitioner.decorateKey(ByteBufferUtil.readWithShortLength(ifile)); - currentEntry = rowIndexEntrySerializer.deserialize(ifile, sstable.descriptor.version); + currentEntry = rowIndexEntrySerializer.deserialize(ifile); } while (!currentRange.contains(currentKey)); } else @@ -283,7 +301,7 @@ public class BigTableScanner implements ISSTableScanner { // we need the position of the start of the next key, regardless of whether it falls in the current range nextKey = sstable.partitioner.decorateKey(ByteBufferUtil.readWithShortLength(ifile)); - nextEntry = rowIndexEntrySerializer.deserialize(ifile, sstable.descriptor.version); + nextEntry = rowIndexEntrySerializer.deserialize(ifile); if (!currentRange.contains(nextKey)) { @@ -292,21 +310,34 @@ public class BigTableScanner implements ISSTableScanner } } - if (dataRange == null || dataRange.selectsFullRowFor(currentKey.getKey())) + /* + * For a given partition key, we want to avoid hitting the data + * file unless we're explicitely asked to. This is important + * for PartitionRangeReadCommand#checkCacheFilter. + */ + return new LazilyInitializedUnfilteredRowIterator(currentKey) { - dfile.seek(currentEntry.position + currentEntry.headerOffset()); - ByteBufferUtil.readWithShortLength(dfile); // key - return new SSTableIdentityIterator(sstable, dfile, currentKey); - } - - return new LazyColumnIterator(currentKey, new IColumnIteratorFactory() - { - public OnDiskAtomIterator create() + protected UnfilteredRowIterator initializeIterator() { - return dataRange.columnFilter(currentKey.getKey()).getSSTableColumnIterator(sstable, dfile, currentKey, currentEntry); - } - }); + try + { + if (dataRange == null) + { + dfile.seek(currentEntry.position + currentEntry.headerOffset()); + ByteBufferUtil.readWithShortLength(dfile); // key + return new SSTableIdentityIterator(sstable, dfile, partitionKey()); + } + ClusteringIndexFilter filter = dataRange.clusteringIndexFilter(partitionKey()); + return filter.filter(sstable.iterator(dfile, partitionKey(), currentEntry, columns, filter.isReversed(), isForThrift)); + } + catch (CorruptSSTableException | IOException e) + { + sstable.markSuspect(); + throw new CorruptSSTableException(e, sstable.getFilename()); + } + } + }; } catch (CorruptSSTableException | IOException e) { @@ -314,6 +345,11 @@ public class BigTableScanner implements ISSTableScanner throw new CorruptSSTableException(e, sstable.getFilename()); } } + + public void close() + { + BigTableScanner.this.close(); + } } @Override @@ -326,7 +362,7 @@ public class BigTableScanner implements ISSTableScanner ")"; } - public static class EmptySSTableScanner implements ISSTableScanner + public static class EmptySSTableScanner extends AbstractUnfilteredPartitionIterator implements ISSTableScanner { private final String filename; @@ -350,20 +386,19 @@ public class BigTableScanner implements ISSTableScanner return filename; } + public boolean isForThrift() + { + return false; + } + public boolean hasNext() { return false; } - public OnDiskAtomIterator next() + public UnfilteredRowIterator next() { return null; } - - public void close() throws IOException { } - - public void remove() { } } - - } diff --git a/src/java/org/apache/cassandra/io/sstable/format/big/BigTableWriter.java b/src/java/org/apache/cassandra/io/sstable/format/big/BigTableWriter.java index 30b55a00ed..66b8ac0d85 100644 --- a/src/java/org/apache/cassandra/io/sstable/format/big/BigTableWriter.java +++ b/src/java/org/apache/cassandra/io/sstable/format/big/BigTableWriter.java @@ -18,10 +18,7 @@ package org.apache.cassandra.io.sstable.format.big; import java.io.*; -import java.nio.ByteBuffer; -import java.util.Collections; -import java.util.Iterator; -import java.util.List; +import java.util.HashSet; import java.util.Map; import java.util.Set; @@ -29,12 +26,13 @@ import org.apache.cassandra.db.*; import org.apache.cassandra.io.sstable.*; import org.apache.cassandra.io.sstable.format.SSTableReader; import org.apache.cassandra.io.sstable.format.SSTableWriter; -import org.apache.cassandra.io.sstable.format.Version; + import org.slf4j.Logger; import org.slf4j.LoggerFactory; import org.apache.cassandra.config.CFMetaData; +import org.apache.cassandra.config.ColumnDefinition; import org.apache.cassandra.config.DatabaseDescriptor; -import org.apache.cassandra.db.compaction.AbstractCompactedRow; +import org.apache.cassandra.db.rows.*; import org.apache.cassandra.dht.IPartitioner; import org.apache.cassandra.io.FSWriteError; import org.apache.cassandra.io.compress.CompressedSequentialWriter; @@ -47,7 +45,6 @@ import org.apache.cassandra.utils.ByteBufferUtil; import org.apache.cassandra.utils.FBUtilities; import org.apache.cassandra.utils.FilterFactory; import org.apache.cassandra.utils.IFilter; -import org.apache.cassandra.utils.StreamingHistogram; import org.apache.cassandra.utils.concurrent.Transactional; import static org.apache.cassandra.utils.Throwables.merge; @@ -66,9 +63,9 @@ public class BigTableWriter extends SSTableWriter private DecoratedKey lastWrittenKey; private FileMark dataMark; - BigTableWriter(Descriptor descriptor, Long keyCount, Long repairedAt, CFMetaData metadata, IPartitioner partitioner, MetadataCollector metadataCollector) + public BigTableWriter(Descriptor descriptor, Long keyCount, Long repairedAt, CFMetaData metadata, IPartitioner partitioner, MetadataCollector metadataCollector, SerializationHeader header) { - super(descriptor, keyCount, repairedAt, metadata, partitioner, metadataCollector); + super(descriptor, keyCount, repairedAt, metadata, partitioner, metadataCollector, header); if (compression) { @@ -124,21 +121,38 @@ public class BigTableWriter extends SSTableWriter } /** - * @param row - * @return null if the row was compacted away entirely; otherwise, the PK index entry for this row + * Appends partition data to this writer. + * + * @param iterator the partition to write + * @return the created index entry if something was written, that is if {@code iterator} + * wasn't empty, {@code null} otherwise. + * + * @throws FSWriteError if a write to the dataFile fails */ - public RowIndexEntry append(AbstractCompactedRow row) + public RowIndexEntry append(UnfilteredRowIterator iterator) { - long startPosition = beforeAppend(row.key); - RowIndexEntry entry; - try + DecoratedKey key = iterator.partitionKey(); + + if (key.getKey().remaining() > FBUtilities.MAX_UNSIGNED_SHORT) { - entry = row.write(startPosition, dataFile); - if (entry == null) - return null; + logger.error("Key size {} exceeds maximum of {}, skipping row", key.getKey().remaining(), FBUtilities.MAX_UNSIGNED_SHORT); + return null; + } + + if (iterator.isEmpty()) + return null; + + long startPosition = beforeAppend(key); + + try (StatsCollector withStats = new StatsCollector(iterator, metadataCollector)) + { + ColumnIndex index = ColumnIndex.writeAndBuildIndex(withStats, dataFile, header, descriptor.version); + + RowIndexEntry entry = RowIndexEntry.create(startPosition, iterator.partitionLevelDeletion(), index); + long endPosition = dataFile.getFilePointer(); - metadataCollector.update(endPosition - startPosition, row.columnStats()); - afterAppend(row.key, endPosition, entry); + metadataCollector.addPartitionSizeInBytes(endPosition - startPosition); + afterAppend(key, endPosition, entry); return entry; } catch (IOException e) @@ -147,130 +161,77 @@ public class BigTableWriter extends SSTableWriter } } - public void append(DecoratedKey decoratedKey, ColumnFamily cf) + private static class StatsCollector extends WrappingUnfilteredRowIterator { - if (decoratedKey.getKey().remaining() > FBUtilities.MAX_UNSIGNED_SHORT) + private int cellCount; + private final MetadataCollector collector; + private final Set complexColumnsSetInRow = new HashSet<>(); + + StatsCollector(UnfilteredRowIterator iter, MetadataCollector collector) { - logger.error("Key size {} exceeds maximum of {}, skipping row", - decoratedKey.getKey().remaining(), - FBUtilities.MAX_UNSIGNED_SHORT); - return; + super(iter); + this.collector = collector; + collector.update(iter.partitionLevelDeletion()); } - long startPosition = beforeAppend(decoratedKey); - long endPosition; - try + @Override + public Unfiltered next() { - RowIndexEntry entry = rawAppend(cf, startPosition, decoratedKey, dataFile.stream); - endPosition = dataFile.getFilePointer(); - afterAppend(decoratedKey, endPosition, entry); - } - catch (IOException e) - { - throw new FSWriteError(e, dataFile.getPath()); - } - metadataCollector.update(endPosition - startPosition, cf.getColumnStats()); - } + Unfiltered unfiltered = super.next(); + collector.updateClusteringValues(unfiltered.clustering()); - private static RowIndexEntry rawAppend(ColumnFamily cf, long startPosition, DecoratedKey key, DataOutputPlus out) throws IOException - { - assert cf.hasColumns() || cf.isMarkedForDelete(); - - ColumnIndex.Builder builder = new ColumnIndex.Builder(cf, key.getKey(), out); - ColumnIndex index = builder.build(cf); - - out.writeShort(END_OF_ROW); - return RowIndexEntry.create(startPosition, cf.deletionInfo().getTopLevelDeletion(), index); - } - - /** - * @throws IOException if a read from the DataInput fails - * @throws FSWriteError if a write to the dataFile fails - */ - public long appendFromStream(DecoratedKey key, CFMetaData metadata, DataInput in, Version version) throws IOException - { - long currentPosition = beforeAppend(key); - - ColumnStats.MaxLongTracker maxTimestampTracker = new ColumnStats.MaxLongTracker(Long.MAX_VALUE); - ColumnStats.MinLongTracker minTimestampTracker = new ColumnStats.MinLongTracker(Long.MIN_VALUE); - ColumnStats.MaxIntTracker maxDeletionTimeTracker = new ColumnStats.MaxIntTracker(Integer.MAX_VALUE); - List minColumnNames = Collections.emptyList(); - List maxColumnNames = Collections.emptyList(); - StreamingHistogram tombstones = new StreamingHistogram(TOMBSTONE_HISTOGRAM_BIN_SIZE); - boolean hasLegacyCounterShards = false; - - ColumnFamily cf = ArrayBackedSortedColumns.factory.create(metadata); - cf.delete(DeletionTime.serializer.deserialize(in)); - - ColumnIndex.Builder columnIndexer = new ColumnIndex.Builder(cf, key.getKey(), dataFile.stream); - - if (cf.deletionInfo().getTopLevelDeletion().localDeletionTime < Integer.MAX_VALUE) - { - tombstones.update(cf.deletionInfo().getTopLevelDeletion().localDeletionTime); - maxDeletionTimeTracker.update(cf.deletionInfo().getTopLevelDeletion().localDeletionTime); - minTimestampTracker.update(cf.deletionInfo().getTopLevelDeletion().markedForDeleteAt); - maxTimestampTracker.update(cf.deletionInfo().getTopLevelDeletion().markedForDeleteAt); - } - - Iterator rangeTombstoneIterator = cf.deletionInfo().rangeIterator(); - while (rangeTombstoneIterator.hasNext()) - { - RangeTombstone rangeTombstone = rangeTombstoneIterator.next(); - tombstones.update(rangeTombstone.getLocalDeletionTime()); - minTimestampTracker.update(rangeTombstone.timestamp()); - maxTimestampTracker.update(rangeTombstone.timestamp()); - maxDeletionTimeTracker.update(rangeTombstone.getLocalDeletionTime()); - minColumnNames = ColumnNameHelper.minComponents(minColumnNames, rangeTombstone.min, metadata.comparator); - maxColumnNames = ColumnNameHelper.maxComponents(maxColumnNames, rangeTombstone.max, metadata.comparator); - } - - Iterator iter = AbstractCell.onDiskIterator(in, ColumnSerializer.Flag.PRESERVE_SIZE, Integer.MIN_VALUE, version, metadata.comparator); - try - { - while (iter.hasNext()) + switch (unfiltered.kind()) { - OnDiskAtom atom = iter.next(); - if (atom == null) + case ROW: + Row row = (Row) unfiltered; + collector.update(row.primaryKeyLivenessInfo()); + collector.update(row.deletion()); + + int simpleColumnsSet = 0; + complexColumnsSetInRow.clear(); + + for (Cell cell : row) + { + if (cell.column().isComplex()) + complexColumnsSetInRow.add(cell.column()); + else + ++simpleColumnsSet; + + ++cellCount; + collector.update(cell.livenessInfo()); + + if (cell.isCounterCell()) + collector.updateHasLegacyCounterShards(CounterCells.hasLegacyShards(cell)); + } + + for (int i = 0; i < row.columns().complexColumnCount(); i++) + collector.update(row.getDeletion(row.columns().getComplex(i))); + + collector.updateColumnSetPerRow(simpleColumnsSet + complexColumnsSetInRow.size()); + + break; + case RANGE_TOMBSTONE_MARKER: + if (((RangeTombstoneMarker) unfiltered).isBoundary()) + { + RangeTombstoneBoundaryMarker bm = (RangeTombstoneBoundaryMarker)unfiltered; + collector.update(bm.endDeletionTime()); + collector.update(bm.startDeletionTime()); + } + else + { + collector.update(((RangeTombstoneBoundMarker)unfiltered).deletionTime()); + } break; - - if (atom instanceof CounterCell) - { - atom = ((CounterCell) atom).markLocalToBeCleared(); - hasLegacyCounterShards = hasLegacyCounterShards || ((CounterCell) atom).hasLegacyShards(); - } - - int deletionTime = atom.getLocalDeletionTime(); - if (deletionTime < Integer.MAX_VALUE) - tombstones.update(deletionTime); - minTimestampTracker.update(atom.timestamp()); - maxTimestampTracker.update(atom.timestamp()); - minColumnNames = ColumnNameHelper.minComponents(minColumnNames, atom.name(), metadata.comparator); - maxColumnNames = ColumnNameHelper.maxComponents(maxColumnNames, atom.name(), metadata.comparator); - maxDeletionTimeTracker.update(atom.getLocalDeletionTime()); - - columnIndexer.add(atom); // This write the atom on disk too } - - columnIndexer.maybeWriteEmptyRowHeader(); - dataFile.stream.writeShort(END_OF_ROW); + return unfiltered; } - catch (IOException e) + + @Override + public void close() { - throw new FSWriteError(e, dataFile.getPath()); + collector.addCellPerPartitionCount(cellCount); + super.close(); } - - metadataCollector.updateMinTimestamp(minTimestampTracker.get()) - .updateMaxTimestamp(maxTimestampTracker.get()) - .updateMaxLocalDeletionTime(maxDeletionTimeTracker.get()) - .addRowSize(dataFile.getFilePointer() - currentPosition) - .addColumnCount(columnIndexer.writtenAtomCount()) - .mergeTombstoneHistogram(tombstones) - .updateMinColumnNames(minColumnNames) - .updateMaxColumnNames(maxColumnNames) - .updateHasLegacyCounterShards(hasLegacyCounterShards); - - afterAppend(key, currentPosition, RowIndexEntry.create(currentPosition, cf.deletionInfo().getTopLevelDeletion(), columnIndexer.build())); - return currentPosition; } private Descriptor makeTmpLinks() @@ -303,7 +264,7 @@ public class BigTableWriter extends SSTableWriter components, metadata, partitioner, ifile, dfile, iwriter.summary.build(partitioner, boundary), - iwriter.bf.sharedCopy(), maxDataAge, stats, SSTableReader.OpenReason.EARLY); + iwriter.bf.sharedCopy(), maxDataAge, stats, SSTableReader.OpenReason.EARLY, header); // now it's open, find the ACTUAL last readable key (i.e. for which the data file has also been flushed) sstable.first = getMinimalKey(first); @@ -339,7 +300,8 @@ public class BigTableWriter extends SSTableWriter iwriter.bf.sharedCopy(), maxDataAge, stats, - openReason); + openReason, + header); sstable.first = getMinimalKey(first); sstable.last = getMinimalKey(last); return sstable; diff --git a/src/java/org/apache/cassandra/io/sstable/format/big/IndexedSliceReader.java b/src/java/org/apache/cassandra/io/sstable/format/big/IndexedSliceReader.java deleted file mode 100644 index 6db9c3df8d..0000000000 --- a/src/java/org/apache/cassandra/io/sstable/format/big/IndexedSliceReader.java +++ /dev/null @@ -1,542 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.apache.cassandra.io.sstable.format.big; - -import java.io.IOException; -import java.util.ArrayDeque; -import java.util.Deque; -import java.util.List; - -import com.google.common.collect.AbstractIterator; - -import org.apache.cassandra.db.*; -import org.apache.cassandra.db.columniterator.OnDiskAtomIterator; -import org.apache.cassandra.db.composites.CellNameType; -import org.apache.cassandra.db.composites.Composite; -import org.apache.cassandra.db.filter.ColumnSlice; -import org.apache.cassandra.io.sstable.CorruptSSTableException; -import org.apache.cassandra.io.sstable.IndexHelper; -import org.apache.cassandra.io.sstable.IndexHelper.IndexInfo; -import org.apache.cassandra.io.sstable.format.SSTableReader; -import org.apache.cassandra.io.util.FileDataInput; -import org.apache.cassandra.io.util.FileMark; -import org.apache.cassandra.tracing.Tracing; -import org.apache.cassandra.utils.ByteBufferUtil; - -/** - * This is a reader that finds the block for a starting column and returns blocks before/after it for each next call. - * This function assumes that the CF is sorted by name and exploits the name index. - */ -class IndexedSliceReader extends AbstractIterator implements OnDiskAtomIterator -{ - private final ColumnFamily emptyColumnFamily; - - private final SSTableReader sstable; - private final List indexes; - private final FileDataInput originalInput; - private FileDataInput file; - private final boolean reversed; - private final ColumnSlice[] slices; - private final BlockFetcher fetcher; - private final Deque blockColumns = new ArrayDeque(); - private final CellNameType comparator; - - // Holds range tombstone in reverse queries. See addColumn() - private final Deque rangeTombstonesReversed; - - /** - * This slice reader assumes that slices are sorted correctly, e.g. that for forward lookup slices are in - * lexicographic order of start elements and that for reverse lookup they are in reverse lexicographic order of - * finish (reverse start) elements. i.e. forward: [a,b],[d,e],[g,h] reverse: [h,g],[e,d],[b,a]. This reader also - * assumes that validation has been performed in terms of intervals (no overlapping intervals). - */ - IndexedSliceReader(SSTableReader sstable, RowIndexEntry indexEntry, FileDataInput input, ColumnSlice[] slices, boolean reversed) - { - Tracing.trace("Seeking to partition indexed section in data file"); - this.sstable = sstable; - this.originalInput = input; - this.reversed = reversed; - this.slices = slices; - this.comparator = sstable.metadata.comparator; - this.rangeTombstonesReversed = reversed ? new ArrayDeque() : null; - - try - { - this.indexes = indexEntry.columnsIndex(); - emptyColumnFamily = ArrayBackedSortedColumns.factory.create(sstable.metadata); - if (indexes.isEmpty()) - { - setToRowStart(indexEntry, input); - emptyColumnFamily.delete(DeletionTime.serializer.deserialize(file)); - fetcher = new SimpleBlockFetcher(); - } - else - { - emptyColumnFamily.delete(indexEntry.deletionTime()); - fetcher = new IndexedBlockFetcher(indexEntry.position); - } - } - catch (IOException e) - { - sstable.markSuspect(); - throw new CorruptSSTableException(e, file.getPath()); - } - } - - /** - * Sets the seek position to the start of the row for column scanning. - */ - private void setToRowStart(RowIndexEntry rowEntry, FileDataInput in) throws IOException - { - if (in == null) - { - this.file = sstable.getFileDataInput(rowEntry.position); - } - else - { - this.file = in; - in.seek(rowEntry.position); - } - sstable.partitioner.decorateKey(ByteBufferUtil.readWithShortLength(file)); - } - - public ColumnFamily getColumnFamily() - { - return emptyColumnFamily; - } - - public DecoratedKey getKey() - { - throw new UnsupportedOperationException(); - } - - protected OnDiskAtom computeNext() - { - while (true) - { - if (reversed) - { - // Return all tombstone for the block first (see addColumn() below) - OnDiskAtom column = rangeTombstonesReversed.poll(); - if (column != null) - return column; - } - - OnDiskAtom column = blockColumns.poll(); - if (column == null) - { - if (!fetcher.fetchMoreData()) - return endOfData(); - } - else - { - return column; - } - } - } - - public void close() throws IOException - { - if (originalInput == null && file != null) - file.close(); - } - - protected void addColumn(OnDiskAtom col) - { - if (reversed) - { - /* - * We put range tomstone markers at the beginning of the range they delete. But for reversed queries, - * the caller still need to know about a RangeTombstone before it sees any column that it covers. - * To make that simple, we keep said tombstones separate and return them all before any column for - * a given block. - */ - if (col instanceof RangeTombstone) - rangeTombstonesReversed.addFirst(col); - else - blockColumns.addFirst(col); - } - else - { - blockColumns.addLast(col); - } - } - - private abstract class BlockFetcher - { - protected int currentSliceIdx; - - protected BlockFetcher(int sliceIdx) - { - this.currentSliceIdx = sliceIdx; - } - - /* - * Return the smallest key selected by the current ColumnSlice. - */ - protected Composite currentStart() - { - return reversed ? slices[currentSliceIdx].finish : slices[currentSliceIdx].start; - } - - /* - * Return the biggest key selected by the current ColumnSlice. - */ - protected Composite currentFinish() - { - return reversed ? slices[currentSliceIdx].start : slices[currentSliceIdx].finish; - } - - protected abstract boolean setNextSlice(); - - protected abstract boolean fetchMoreData(); - - protected boolean isColumnBeforeSliceStart(OnDiskAtom column) - { - return isBeforeSliceStart(column.name()); - } - - protected boolean isBeforeSliceStart(Composite name) - { - Composite start = currentStart(); - return !start.isEmpty() && comparator.compare(name, start) < 0; - } - - protected boolean isColumnBeforeSliceFinish(OnDiskAtom column) - { - Composite finish = currentFinish(); - return finish.isEmpty() || comparator.compare(column.name(), finish) <= 0; - } - - protected boolean isAfterSliceFinish(Composite name) - { - Composite finish = currentFinish(); - return !finish.isEmpty() && comparator.compare(name, finish) > 0; - } - } - - private class IndexedBlockFetcher extends BlockFetcher - { - // where this row starts - private final long columnsStart; - - // the index entry for the next block to deserialize - private int nextIndexIdx = -1; - - // index of the last block we've read from disk; - private int lastDeserializedBlock = -1; - - // For reversed, keep columns at the beginning of the last deserialized block that - // may still match a slice - private final Deque prefetched; - - public IndexedBlockFetcher(long columnsStart) - { - super(-1); - this.columnsStart = columnsStart; - this.prefetched = reversed ? new ArrayDeque() : null; - setNextSlice(); - } - - protected boolean setNextSlice() - { - while (++currentSliceIdx < slices.length) - { - nextIndexIdx = IndexHelper.indexFor(slices[currentSliceIdx].start, indexes, comparator, reversed, nextIndexIdx); - if (nextIndexIdx < 0 || nextIndexIdx >= indexes.size()) - // no index block for that slice - continue; - - // Check if we can exclude this slice entirely from the index - IndexInfo info = indexes.get(nextIndexIdx); - if (reversed) - { - if (!isBeforeSliceStart(info.lastName)) - return true; - } - else - { - if (!isAfterSliceFinish(info.firstName)) - return true; - } - } - nextIndexIdx = -1; - return false; - } - - protected boolean hasMoreSlice() - { - return currentSliceIdx < slices.length; - } - - protected boolean fetchMoreData() - { - if (!hasMoreSlice()) - return false; - - // If we read blocks in reversed disk order, we may have columns from the previous block to handle. - // Note that prefetched keeps columns in reversed disk order. - // Also note that Range Tombstone handling is a bit tricky, because we may run into range tombstones - // that cover a slice *after* we've move to the previous slice. To keep it simple, we simply include - // every RT in prefetched: it's only slightly inefficient to do so and there is only so much RT that - // can be mistakenly added this way. - if (reversed && !prefetched.isEmpty()) - { - // Avoids some comparison when we know it's not useful - boolean inSlice = false; - - OnDiskAtom prefetchedCol; - while ((prefetchedCol = prefetched.peek()) != null) - { - // col is before slice, we update the slice - if (isColumnBeforeSliceStart(prefetchedCol)) - { - inSlice = false; - - // As explained above, we add RT unconditionally - if (prefetchedCol instanceof RangeTombstone) - { - blockColumns.addLast(prefetched.poll()); - continue; - } - - // Otherwise, we either move to the next slice. If we have no more slice, then - // simply unwind prefetched entirely and add all RT. - if (!setNextSlice()) - { - while ((prefetchedCol = prefetched.poll()) != null) - if (prefetchedCol instanceof RangeTombstone) - blockColumns.addLast(prefetchedCol); - break; - } - - } - // col is within slice, all columns - // (we go in reverse, so as soon as we are in a slice, no need to check - // we're after the slice until we change slice) - else if (inSlice || isColumnBeforeSliceFinish(prefetchedCol)) - { - blockColumns.addLast(prefetched.poll()); - inSlice = true; - } - // if col is after slice, ignore - else - { - prefetched.poll(); - } - } - - if (!blockColumns.isEmpty()) - return true; - else if (!hasMoreSlice()) - return false; - } - try - { - return getNextBlock(); - } - catch (IOException e) - { - throw new CorruptSSTableException(e, file.getPath()); - } - } - - private boolean getNextBlock() throws IOException - { - if (lastDeserializedBlock == nextIndexIdx) - { - if (reversed) - nextIndexIdx--; - else - nextIndexIdx++; - } - lastDeserializedBlock = nextIndexIdx; - - // Are we done? - if (lastDeserializedBlock < 0 || lastDeserializedBlock >= indexes.size()) - return false; - - IndexInfo currentIndex = indexes.get(lastDeserializedBlock); - - /* seek to the correct offset to the data, and calculate the data size */ - long positionToSeek = columnsStart + currentIndex.offset; - - // With new promoted indexes, our first seek in the data file will happen at that point. - if (file == null) - file = originalInput == null ? sstable.getFileDataInput(positionToSeek) : originalInput; - - AtomDeserializer deserializer = emptyColumnFamily.metadata().getOnDiskDeserializer(file, sstable.descriptor.version); - - file.seek(positionToSeek); - FileMark mark = file.mark(); - - // We remenber when we are whithin a slice to avoid some comparison - boolean inSlice = false; - - // scan from index start - while (file.bytesPastMark(mark) < currentIndex.width || deserializer.hasUnprocessed()) - { - // col is before slice - // (If in slice, don't bother checking that until we change slice) - Composite start = currentStart(); - if (!inSlice && !start.isEmpty() && deserializer.compareNextTo(start) < 0) - { - // If it's a rangeTombstone, then we need to read it and include it unless it's end - // stops before our slice start. - if (deserializer.nextIsRangeTombstone()) - { - RangeTombstone rt = (RangeTombstone)deserializer.readNext(); - if (comparator.compare(rt.max, start) >= 0) - addColumn(rt); - continue; - } - - if (reversed) - { - // the next slice select columns that are before the current one, so it may - // match this column, so keep it around. - prefetched.addFirst(deserializer.readNext()); - } - else - { - deserializer.skipNext(); - } - } - // col is within slice - else - { - Composite finish = currentFinish(); - if (finish.isEmpty() || deserializer.compareNextTo(finish) <= 0) - { - inSlice = true; - addColumn(deserializer.readNext()); - } - // col is after slice. - else - { - // When reading forward, if we hit a column that sorts after the current slice, it means we're done with this slice. - // For reversed, this may either mean that we're done with the current slice, or that we need to read the previous - // index block. However, we can be sure that we are in the first case though (the current slice is done) if the first - // columns of the block were not part of the current slice, i.e. if we have columns in prefetched. - if (reversed && prefetched.isEmpty()) - break; - - if (!setNextSlice()) - break; - - inSlice = false; - - // The next index block now corresponds to the first block that may have columns for the newly set slice. - // So if it's different from the current block, we're done with this block. And in that case, we know - // that our prefetched columns won't match. - if (nextIndexIdx != lastDeserializedBlock) - { - if (reversed) - prefetched.clear(); - break; - } - - // Even if the next slice may have column in this blocks, if we're reversed, those columns have been - // prefetched and we're done with that block - if (reversed) - break; - - // otherwise, we will deal with that column at the next iteration - } - } - } - return true; - } - } - - private class SimpleBlockFetcher extends BlockFetcher - { - public SimpleBlockFetcher() throws IOException - { - // Since we have to deserialize in order and will read all slices might as well reverse the slices and - // behave as if it was not reversed - super(reversed ? slices.length - 1 : 0); - - // We remenber when we are whithin a slice to avoid some comparison - boolean inSlice = false; - - AtomDeserializer deserializer = emptyColumnFamily.metadata().getOnDiskDeserializer(file, sstable.descriptor.version); - while (deserializer.hasNext()) - { - // col is before slice - // (If in slice, don't bother checking that until we change slice) - Composite start = currentStart(); - if (!inSlice && !start.isEmpty() && deserializer.compareNextTo(start) < 0) - { - // If it's a rangeTombstone, then we need to read it and include it unless it's end - // stops before our slice start. Otherwise, we can skip it. - if (deserializer.nextIsRangeTombstone()) - { - RangeTombstone rt = (RangeTombstone)deserializer.readNext(); - if (comparator.compare(rt.max, start) >= 0) - addColumn(rt); - } - else - { - deserializer.skipNext(); - } - continue; - } - - // col is within slice - Composite finish = currentFinish(); - if (finish.isEmpty() || deserializer.compareNextTo(finish) <= 0) - { - inSlice = true; - addColumn(deserializer.readNext()); - } - // col is after slice. more slices? - else - { - inSlice = false; - if (!setNextSlice()) - break; - } - } - } - - protected boolean setNextSlice() - { - if (reversed) - { - if (currentSliceIdx <= 0) - return false; - - currentSliceIdx--; - } - else - { - if (currentSliceIdx >= slices.length - 1) - return false; - - currentSliceIdx++; - } - return true; - } - - protected boolean fetchMoreData() - { - return false; - } - } -} diff --git a/src/java/org/apache/cassandra/io/sstable/format/big/SSTableNamesIterator.java b/src/java/org/apache/cassandra/io/sstable/format/big/SSTableNamesIterator.java deleted file mode 100644 index b8910c772f..0000000000 --- a/src/java/org/apache/cassandra/io/sstable/format/big/SSTableNamesIterator.java +++ /dev/null @@ -1,264 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.apache.cassandra.io.sstable.format.big; - -import java.io.IOException; -import java.util.*; - -import com.google.common.collect.AbstractIterator; - -import org.apache.cassandra.config.CFMetaData; -import org.apache.cassandra.db.*; -import org.apache.cassandra.db.columniterator.OnDiskAtomIterator; -import org.apache.cassandra.db.composites.CellName; -import org.apache.cassandra.db.composites.CellNameType; -import org.apache.cassandra.io.sstable.CorruptSSTableException; -import org.apache.cassandra.io.sstable.IndexHelper; -import org.apache.cassandra.io.sstable.format.SSTableReader; -import org.apache.cassandra.io.util.FileDataInput; -import org.apache.cassandra.io.util.FileMark; -import org.apache.cassandra.io.util.FileUtils; -import org.apache.cassandra.utils.ByteBufferUtil; - -class SSTableNamesIterator extends AbstractIterator implements OnDiskAtomIterator -{ - private ColumnFamily cf; - private final SSTableReader sstable; - private FileDataInput fileToClose; - private Iterator iter; - public final SortedSet columns; - public final DecoratedKey key; - - public SSTableNamesIterator(SSTableReader sstable, DecoratedKey key, SortedSet columns) - { - assert columns != null; - this.sstable = sstable; - this.columns = columns; - this.key = key; - - RowIndexEntry indexEntry = sstable.getPosition(key, SSTableReader.Operator.EQ); - if (indexEntry == null) - return; - - try - { - read(sstable, null, indexEntry); - } - catch (IOException e) - { - sstable.markSuspect(); - throw new CorruptSSTableException(e, sstable.getFilename()); - } - finally - { - if (fileToClose != null) - FileUtils.closeQuietly(fileToClose); - } - } - - public SSTableNamesIterator(SSTableReader sstable, FileDataInput file, DecoratedKey key, SortedSet columns, RowIndexEntry indexEntry) - { - assert columns != null; - this.sstable = sstable; - this.columns = columns; - this.key = key; - - try - { - read(sstable, file, indexEntry); - } - catch (IOException e) - { - sstable.markSuspect(); - throw new CorruptSSTableException(e, sstable.getFilename()); - } - } - - private FileDataInput createFileDataInput(long position) - { - fileToClose = sstable.getFileDataInput(position); - return fileToClose; - } - - @SuppressWarnings("resource") - private void read(SSTableReader sstable, FileDataInput file, RowIndexEntry indexEntry) - throws IOException - { - List indexList; - - // If the entry is not indexed or the index is not promoted, read from the row start - if (!indexEntry.isIndexed()) - { - if (file == null) - file = createFileDataInput(indexEntry.position); - else - file.seek(indexEntry.position); - - DecoratedKey keyInDisk = sstable.partitioner.decorateKey(ByteBufferUtil.readWithShortLength(file)); - assert keyInDisk.equals(key) : String.format("%s != %s in %s", keyInDisk, key, file.getPath()); - } - - indexList = indexEntry.columnsIndex(); - - if (!indexEntry.isIndexed()) - { - ColumnFamilySerializer serializer = ColumnFamily.serializer; - try - { - cf = ArrayBackedSortedColumns.factory.create(sstable.metadata); - cf.delete(DeletionTime.serializer.deserialize(file)); - } - catch (Exception e) - { - throw new IOException(serializer + " failed to deserialize " + sstable.getColumnFamilyName() + " with " + sstable.metadata + " from " + file, e); - } - } - else - { - cf = ArrayBackedSortedColumns.factory.create(sstable.metadata); - cf.delete(indexEntry.deletionTime()); - } - - List result = new ArrayList(); - if (indexList.isEmpty()) - { - readSimpleColumns(file, columns, result); - } - else - { - readIndexedColumns(sstable.metadata, file, columns, indexList, indexEntry.position, result); - } - - // create an iterator view of the columns we read - iter = result.iterator(); - } - - private void readSimpleColumns(FileDataInput file, SortedSet columnNames, List result) - { - Iterator atomIterator = cf.metadata().getOnDiskIterator(file, sstable.descriptor.version); - int n = 0; - while (atomIterator.hasNext()) - { - OnDiskAtom column = atomIterator.next(); - if (column instanceof Cell) - { - if (columnNames.contains(column.name())) - { - result.add(column); - if (++n >= columns.size()) - break; - } - } - else - { - result.add(column); - } - } - } - - @SuppressWarnings("resource") - private void readIndexedColumns(CFMetaData metadata, - FileDataInput file, - SortedSet columnNames, - List indexList, - long basePosition, - List result) - throws IOException - { - /* get the various column ranges we have to read */ - CellNameType comparator = metadata.comparator; - List ranges = new ArrayList(); - int lastIndexIdx = -1; - for (CellName name : columnNames) - { - int index = IndexHelper.indexFor(name, indexList, comparator, false, lastIndexIdx); - if (index < 0 || index == indexList.size()) - continue; - IndexHelper.IndexInfo indexInfo = indexList.get(index); - // Check the index block does contain the column names and that we haven't inserted this block yet. - if (comparator.compare(name, indexInfo.firstName) < 0 || index == lastIndexIdx) - continue; - - ranges.add(indexInfo); - lastIndexIdx = index; - } - - if (ranges.isEmpty()) - return; - - Iterator toFetch = columnNames.iterator(); - CellName nextToFetch = toFetch.next(); - for (IndexHelper.IndexInfo indexInfo : ranges) - { - long positionToSeek = basePosition + indexInfo.offset; - - // With new promoted indexes, our first seek in the data file will happen at that point. - if (file == null) - file = createFileDataInput(positionToSeek); - - AtomDeserializer deserializer = cf.metadata().getOnDiskDeserializer(file, sstable.descriptor.version); - file.seek(positionToSeek); - FileMark mark = file.mark(); - while (file.bytesPastMark(mark) < indexInfo.width && nextToFetch != null) - { - int cmp = deserializer.compareNextTo(nextToFetch); - if (cmp < 0) - { - // If it's a rangeTombstone, then we need to read it and include - // it if it includes our target. Otherwise, we can skip it. - if (deserializer.nextIsRangeTombstone()) - { - RangeTombstone rt = (RangeTombstone)deserializer.readNext(); - if (comparator.compare(rt.max, nextToFetch) >= 0) - result.add(rt); - } - else - { - deserializer.skipNext(); - } - } - else if (cmp == 0) - { - nextToFetch = toFetch.hasNext() ? toFetch.next() : null; - result.add(deserializer.readNext()); - } - else - nextToFetch = toFetch.hasNext() ? toFetch.next() : null; - } - } - } - - public DecoratedKey getKey() - { - return key; - } - - public ColumnFamily getColumnFamily() - { - return cf; - } - - protected OnDiskAtom computeNext() - { - if (iter == null || !iter.hasNext()) - return endOfData(); - return iter.next(); - } - - public void close() throws IOException { } -} diff --git a/src/java/org/apache/cassandra/io/sstable/format/big/SSTableSliceIterator.java b/src/java/org/apache/cassandra/io/sstable/format/big/SSTableSliceIterator.java deleted file mode 100644 index 07d867d8c4..0000000000 --- a/src/java/org/apache/cassandra/io/sstable/format/big/SSTableSliceIterator.java +++ /dev/null @@ -1,102 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.apache.cassandra.io.sstable.format.big; - -import java.io.IOException; - -import org.apache.cassandra.db.ColumnFamily; -import org.apache.cassandra.db.DecoratedKey; -import org.apache.cassandra.db.OnDiskAtom; -import org.apache.cassandra.db.RowIndexEntry; -import org.apache.cassandra.db.columniterator.OnDiskAtomIterator; -import org.apache.cassandra.db.filter.ColumnSlice; -import org.apache.cassandra.io.sstable.format.SSTableReader; -import org.apache.cassandra.io.util.FileDataInput; - -/** - * A Cell Iterator over SSTable - */ -class SSTableSliceIterator implements OnDiskAtomIterator -{ - private final OnDiskAtomIterator reader; - private final DecoratedKey key; - - public SSTableSliceIterator(SSTableReader sstable, DecoratedKey key, ColumnSlice[] slices, boolean reversed) - { - this.key = key; - RowIndexEntry indexEntry = sstable.getPosition(key, SSTableReader.Operator.EQ); - this.reader = indexEntry == null ? null : createReader(sstable, indexEntry, null, slices, reversed); - } - - /** - * An iterator for a slice within an SSTable - * @param sstable Keyspace for the CFS we are reading from - * @param file Optional parameter that input is read from. If null is passed, this class creates an appropriate one automatically. - * If this class creates, it will close the underlying file when #close() is called. - * If a caller passes a non-null argument, this class will NOT close the underlying file when the iterator is closed (i.e. the caller is responsible for closing the file) - * In all cases the caller should explicitly #close() this iterator. - * @param key The key the requested slice resides under - * @param slices the column slices - * @param reversed Results are returned in reverse order iff reversed is true. - * @param indexEntry position of the row - */ - public SSTableSliceIterator(SSTableReader sstable, FileDataInput file, DecoratedKey key, ColumnSlice[] slices, boolean reversed, RowIndexEntry indexEntry) - { - this.key = key; - reader = createReader(sstable, indexEntry, file, slices, reversed); - } - - private static OnDiskAtomIterator createReader(SSTableReader sstable, RowIndexEntry indexEntry, FileDataInput file, ColumnSlice[] slices, boolean reversed) - { - return slices.length == 1 && slices[0].start.isEmpty() && !reversed - ? new SimpleSliceReader(sstable, indexEntry, file, slices[0].finish) - : new IndexedSliceReader(sstable, indexEntry, file, slices, reversed); - } - - public DecoratedKey getKey() - { - return key; - } - - public ColumnFamily getColumnFamily() - { - return reader == null ? null : reader.getColumnFamily(); - } - - public boolean hasNext() - { - return reader != null && reader.hasNext(); - } - - public OnDiskAtom next() - { - return reader.next(); - } - - public void remove() - { - throw new UnsupportedOperationException(); - } - - public void close() throws IOException - { - if (reader != null) - reader.close(); - } - -} diff --git a/src/java/org/apache/cassandra/io/sstable/format/big/SimpleSliceReader.java b/src/java/org/apache/cassandra/io/sstable/format/big/SimpleSliceReader.java deleted file mode 100644 index 9fec303462..0000000000 --- a/src/java/org/apache/cassandra/io/sstable/format/big/SimpleSliceReader.java +++ /dev/null @@ -1,108 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.apache.cassandra.io.sstable.format.big; - -import java.io.IOException; -import java.util.Iterator; - -import com.google.common.collect.AbstractIterator; -import org.apache.cassandra.db.columniterator.OnDiskAtomIterator; -import org.apache.cassandra.io.sstable.format.SSTableReader; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; - -import org.apache.cassandra.db.*; -import org.apache.cassandra.db.composites.CellNameType; -import org.apache.cassandra.db.composites.Composite; -import org.apache.cassandra.io.sstable.CorruptSSTableException; -import org.apache.cassandra.io.util.FileDataInput; -import org.apache.cassandra.tracing.Tracing; -import org.apache.cassandra.utils.ByteBufferUtil; - -class SimpleSliceReader extends AbstractIterator implements OnDiskAtomIterator -{ - private static final Logger logger = LoggerFactory.getLogger(SimpleSliceReader.class); - - private final FileDataInput file; - private final boolean needsClosing; - private final Composite finishColumn; - private final CellNameType comparator; - private final ColumnFamily emptyColumnFamily; - private final Iterator atomIterator; - - SimpleSliceReader(SSTableReader sstable, RowIndexEntry indexEntry, FileDataInput input, Composite finishColumn) - { - Tracing.trace("Seeking to partition beginning in data file"); - this.finishColumn = finishColumn; - this.comparator = sstable.metadata.comparator; - try - { - if (input == null) - { - this.file = sstable.getFileDataInput(indexEntry.position); - this.needsClosing = true; - } - else - { - this.file = input; - input.seek(indexEntry.position); - this.needsClosing = false; - } - - // Skip key and data size - ByteBufferUtil.skipShortLength(file); - - emptyColumnFamily = ArrayBackedSortedColumns.factory.create(sstable.metadata); - emptyColumnFamily.delete(DeletionTime.serializer.deserialize(file)); - atomIterator = emptyColumnFamily.metadata().getOnDiskIterator(file, sstable.descriptor.version); - } - catch (IOException e) - { - sstable.markSuspect(); - throw new CorruptSSTableException(e, sstable.getFilename()); - } - } - - protected OnDiskAtom computeNext() - { - if (!atomIterator.hasNext()) - return endOfData(); - - OnDiskAtom column = atomIterator.next(); - if (!finishColumn.isEmpty() && comparator.compare(column.name(), finishColumn) > 0) - return endOfData(); - - return column; - } - - public ColumnFamily getColumnFamily() - { - return emptyColumnFamily; - } - - public void close() throws IOException - { - if (needsClosing) - file.close(); - } - - public DecoratedKey getKey() - { - throw new UnsupportedOperationException(); - } -} diff --git a/src/java/org/apache/cassandra/io/sstable/metadata/LegacyMetadataSerializer.java b/src/java/org/apache/cassandra/io/sstable/metadata/LegacyMetadataSerializer.java index 4bd060ed8c..90a9f244b6 100644 --- a/src/java/org/apache/cassandra/io/sstable/metadata/LegacyMetadataSerializer.java +++ b/src/java/org/apache/cassandra/io/sstable/metadata/LegacyMetadataSerializer.java @@ -23,6 +23,7 @@ import java.util.*; import com.google.common.collect.Maps; +import org.apache.cassandra.config.CFMetaData; import org.apache.cassandra.db.commitlog.ReplayPosition; import org.apache.cassandra.io.sstable.Component; import org.apache.cassandra.io.sstable.Descriptor; @@ -64,12 +65,12 @@ public class LegacyMetadataSerializer extends MetadataSerializer out.writeInt(g); StreamingHistogram.serializer.serialize(stats.estimatedTombstoneDropTime, out); out.writeInt(stats.sstableLevel); - out.writeInt(stats.minColumnNames.size()); - for (ByteBuffer columnName : stats.minColumnNames) - ByteBufferUtil.writeWithShortLength(columnName, out); - out.writeInt(stats.maxColumnNames.size()); - for (ByteBuffer columnName : stats.maxColumnNames) - ByteBufferUtil.writeWithShortLength(columnName, out); + out.writeInt(stats.minClusteringValues.size()); + for (ByteBuffer value : stats.minClusteringValues) + ByteBufferUtil.writeWithShortLength(value, out); + out.writeInt(stats.maxClusteringValues.size()); + for (ByteBuffer value : stats.maxClusteringValues) + ByteBufferUtil.writeWithShortLength(value, out); } /** @@ -127,14 +128,19 @@ public class LegacyMetadataSerializer extends MetadataSerializer replayPosition, minTimestamp, maxTimestamp, + Integer.MAX_VALUE, maxLocalDeletionTime, + 0, + Integer.MAX_VALUE, compressionRatio, tombstoneHistogram, sstableLevel, minColumnNames, maxColumnNames, true, - ActiveRepairService.UNREPAIRED_SSTABLE)); + ActiveRepairService.UNREPAIRED_SSTABLE, + -1, + -1)); if (types.contains(MetadataType.COMPACTION)) components.put(MetadataType.COMPACTION, new CompactionMetadata(ancestors, null)); diff --git a/src/java/org/apache/cassandra/io/sstable/metadata/MetadataCollector.java b/src/java/org/apache/cassandra/io/sstable/metadata/MetadataCollector.java index 5962a46813..2574c6212b 100644 --- a/src/java/org/apache/cassandra/io/sstable/metadata/MetadataCollector.java +++ b/src/java/org/apache/cassandra/io/sstable/metadata/MetadataCollector.java @@ -19,22 +19,20 @@ package org.apache.cassandra.io.sstable.metadata; import java.io.File; import java.nio.ByteBuffer; -import java.util.Collection; +import java.util.ArrayList; import java.util.Collections; import java.util.HashSet; import java.util.List; import java.util.Map; import java.util.Set; -import com.google.common.collect.ImmutableList; import com.google.common.collect.Maps; import com.clearspring.analytics.stream.cardinality.HyperLogLogPlus; import com.clearspring.analytics.stream.cardinality.ICardinality; +import org.apache.cassandra.db.*; import org.apache.cassandra.db.commitlog.ReplayPosition; -import org.apache.cassandra.db.composites.CellNameType; -import org.apache.cassandra.io.sstable.ColumnNameHelper; -import org.apache.cassandra.io.sstable.ColumnStats; +import org.apache.cassandra.db.marshal.AbstractType; import org.apache.cassandra.io.sstable.Component; import org.apache.cassandra.io.sstable.SSTable; import org.apache.cassandra.io.sstable.format.SSTableReader; @@ -47,13 +45,13 @@ public class MetadataCollector { public static final double NO_COMPRESSION_RATIO = -1.0; - static EstimatedHistogram defaultColumnCountHistogram() + static EstimatedHistogram defaultCellPerPartitionCountHistogram() { // EH of 114 can track a max value of 2395318855, i.e., > 2B columns return new EstimatedHistogram(114); } - static EstimatedHistogram defaultRowSizeHistogram() + static EstimatedHistogram defaultPartitionSizeHistogram() { // EH of 150 can track a max value of 1697806495183, i.e., > 1.5PB return new EstimatedHistogram(150); @@ -66,34 +64,42 @@ public class MetadataCollector public static StatsMetadata defaultStatsMetadata() { - return new StatsMetadata(defaultRowSizeHistogram(), - defaultColumnCountHistogram(), + return new StatsMetadata(defaultPartitionSizeHistogram(), + defaultCellPerPartitionCountHistogram(), ReplayPosition.NONE, Long.MIN_VALUE, Long.MAX_VALUE, Integer.MAX_VALUE, + Integer.MAX_VALUE, + 0, + Integer.MAX_VALUE, NO_COMPRESSION_RATIO, defaultTombstoneDropTimeHistogram(), 0, Collections.emptyList(), Collections.emptyList(), true, - ActiveRepairService.UNREPAIRED_SSTABLE); + ActiveRepairService.UNREPAIRED_SSTABLE, + -1, + -1); } - protected EstimatedHistogram estimatedRowSize = defaultRowSizeHistogram(); - protected EstimatedHistogram estimatedColumnCount = defaultColumnCountHistogram(); + protected EstimatedHistogram estimatedPartitionSize = defaultPartitionSizeHistogram(); + // TODO: cound the number of row per partition (either with the number of cells, or instead) + protected EstimatedHistogram estimatedCellPerPartitionCount = defaultCellPerPartitionCountHistogram(); protected ReplayPosition replayPosition = ReplayPosition.NONE; - protected long minTimestamp = Long.MAX_VALUE; - protected long maxTimestamp = Long.MIN_VALUE; - protected int maxLocalDeletionTime = Integer.MIN_VALUE; + protected final MinMaxLongTracker timestampTracker = new MinMaxLongTracker(); + protected final MinMaxIntTracker localDeletionTimeTracker = new MinMaxIntTracker(LivenessInfo.NO_DELETION_TIME, LivenessInfo.NO_DELETION_TIME); + protected final MinMaxIntTracker ttlTracker = new MinMaxIntTracker(LivenessInfo.NO_TTL, LivenessInfo.NO_TTL); protected double compressionRatio = NO_COMPRESSION_RATIO; protected Set ancestors = new HashSet<>(); protected StreamingHistogram estimatedTombstoneDropTime = defaultTombstoneDropTimeHistogram(); protected int sstableLevel; - protected List minColumnNames = Collections.emptyList(); - protected List maxColumnNames = Collections.emptyList(); + protected ByteBuffer[] minClusteringValues; + protected ByteBuffer[] maxClusteringValues; protected boolean hasLegacyCounterShards = false; + protected long totalColumnsSet; + protected long totalRows; /** * Default cardinality estimation method is to use HyperLogLog++. @@ -102,16 +108,19 @@ public class MetadataCollector * See CASSANDRA-5906 for detail. */ protected ICardinality cardinality = new HyperLogLogPlus(13, 25); - private final CellNameType columnNameComparator; + private final ClusteringComparator comparator; - public MetadataCollector(CellNameType columnNameComparator) + public MetadataCollector(ClusteringComparator comparator) { - this.columnNameComparator = columnNameComparator; + this.comparator = comparator; + + this.minClusteringValues = new ByteBuffer[comparator.size()]; + this.maxClusteringValues = new ByteBuffer[comparator.size()]; } - public MetadataCollector(Iterable sstables, CellNameType columnNameComparator, int level, boolean skipAncestors) + public MetadataCollector(Iterable sstables, ClusteringComparator comparator, int level, boolean skipAncestors) { - this(columnNameComparator); + this(comparator); replayPosition(ReplayPosition.getReplayPosition(sstables)); sstableLevel(level); @@ -129,9 +138,9 @@ public class MetadataCollector } } - public MetadataCollector(Iterable sstables, CellNameType columnNameComparator, int level) + public MetadataCollector(Iterable sstables, ClusteringComparator comparator, int level) { - this(sstables, columnNameComparator, level, false); + this(sstables, comparator, level, false); } public MetadataCollector addKey(ByteBuffer key) @@ -141,15 +150,15 @@ public class MetadataCollector return this; } - public MetadataCollector addRowSize(long rowSize) + public MetadataCollector addPartitionSizeInBytes(long partitionSize) { - estimatedRowSize.add(rowSize); + estimatedPartitionSize.add(partitionSize); return this; } - public MetadataCollector addColumnCount(long columnCount) + public MetadataCollector addCellPerPartitionCount(long cellCount) { - estimatedColumnCount.add(columnCount); + estimatedCellPerPartitionCount.add(cellCount); return this; } @@ -169,34 +178,50 @@ public class MetadataCollector return this; } - public MetadataCollector updateMinTimestamp(long potentialMin) + public MetadataCollector update(LivenessInfo newInfo) { - minTimestamp = Math.min(minTimestamp, potentialMin); + // If the info doesn't have a timestamp, this means the info is basically irrelevant (it's a row + // update whose only info we care are the cells info basically). + if (newInfo.hasTimestamp()) + { + updateTimestamp(newInfo.timestamp()); + updateTTL(newInfo.ttl()); + updateLocalDeletionTime(newInfo.localDeletionTime()); + } return this; } - public MetadataCollector updateMaxTimestamp(long potentialMax) + public MetadataCollector update(DeletionTime dt) { - maxTimestamp = Math.max(maxTimestamp, potentialMax); + if (!dt.isLive()) + { + updateTimestamp(dt.markedForDeleteAt()); + updateLocalDeletionTime(dt.localDeletionTime()); + } return this; } - public MetadataCollector updateMaxLocalDeletionTime(int maxLocalDeletionTime) + public MetadataCollector updateColumnSetPerRow(long columnSetInRow) { - this.maxLocalDeletionTime = Math.max(this.maxLocalDeletionTime, maxLocalDeletionTime); + totalColumnsSet += columnSetInRow; + ++totalRows; return this; } - public MetadataCollector estimatedRowSize(EstimatedHistogram estimatedRowSize) + private void updateTimestamp(long newTimestamp) { - this.estimatedRowSize = estimatedRowSize; - return this; + timestampTracker.update(newTimestamp); } - public MetadataCollector estimatedColumnCount(EstimatedHistogram estimatedColumnCount) + private void updateLocalDeletionTime(int newLocalDeletionTime) { - this.estimatedColumnCount = estimatedColumnCount; - return this; + localDeletionTimeTracker.update(newLocalDeletionTime); + estimatedTombstoneDropTime.update(newLocalDeletionTime); + } + + private void updateTTL(int newTTL) + { + ttlTracker.update(newTTL); } public MetadataCollector replayPosition(ReplayPosition replayPosition) @@ -217,18 +242,41 @@ public class MetadataCollector return this; } - public MetadataCollector updateMinColumnNames(List minColumnNames) + public MetadataCollector updateClusteringValues(ClusteringPrefix clustering) { - if (minColumnNames.size() > 0) - this.minColumnNames = ColumnNameHelper.mergeMin(this.minColumnNames, minColumnNames, columnNameComparator); + int size = clustering.size(); + for (int i = 0; i < size; i++) + { + AbstractType type = comparator.subtype(i); + ByteBuffer newValue = clustering.get(i); + minClusteringValues[i] = min(minClusteringValues[i], newValue, type); + maxClusteringValues[i] = max(maxClusteringValues[i], newValue, type); + } return this; } - public MetadataCollector updateMaxColumnNames(List maxColumnNames) + private static ByteBuffer min(ByteBuffer b1, ByteBuffer b2, AbstractType comparator) { - if (maxColumnNames.size() > 0) - this.maxColumnNames = ColumnNameHelper.mergeMax(this.maxColumnNames, maxColumnNames, columnNameComparator); - return this; + if (b1 == null) + return b2; + if (b2 == null) + return b1; + + if (comparator.compare(b1, b2) >= 0) + return b2; + return b1; + } + + private static ByteBuffer max(ByteBuffer b1, ByteBuffer b2, AbstractType comparator) + { + if (b1 == null) + return b2; + if (b2 == null) + return b1; + + if (comparator.compare(b1, b2) >= 0) + return b1; + return b2; } public MetadataCollector updateHasLegacyCounterShards(boolean hasLegacyCounterShards) @@ -237,38 +285,136 @@ public class MetadataCollector return this; } - public MetadataCollector update(long rowSize, ColumnStats stats) - { - updateMinTimestamp(stats.minTimestamp); - updateMaxTimestamp(stats.maxTimestamp); - updateMaxLocalDeletionTime(stats.maxLocalDeletionTime); - addRowSize(rowSize); - addColumnCount(stats.columnCount); - mergeTombstoneHistogram(stats.tombstoneHistogram); - updateMinColumnNames(stats.minColumnNames); - updateMaxColumnNames(stats.maxColumnNames); - updateHasLegacyCounterShards(stats.hasLegacyCounterShards); - return this; - } - - public Map finalizeMetadata(String partitioner, double bloomFilterFPChance, long repairedAt) + public Map finalizeMetadata(String partitioner, double bloomFilterFPChance, long repairedAt, SerializationHeader header) { Map components = Maps.newHashMap(); components.put(MetadataType.VALIDATION, new ValidationMetadata(partitioner, bloomFilterFPChance)); - components.put(MetadataType.STATS, new StatsMetadata(estimatedRowSize, - estimatedColumnCount, + components.put(MetadataType.STATS, new StatsMetadata(estimatedPartitionSize, + estimatedCellPerPartitionCount, replayPosition, - minTimestamp, - maxTimestamp, - maxLocalDeletionTime, + timestampTracker.min(), + timestampTracker.max(), + localDeletionTimeTracker.min(), + localDeletionTimeTracker.max(), + ttlTracker.min(), + ttlTracker.max(), compressionRatio, estimatedTombstoneDropTime, sstableLevel, - ImmutableList.copyOf(minColumnNames), - ImmutableList.copyOf(maxColumnNames), + makeList(minClusteringValues), + makeList(maxClusteringValues), hasLegacyCounterShards, - repairedAt)); + repairedAt, + totalColumnsSet, + totalRows)); components.put(MetadataType.COMPACTION, new CompactionMetadata(ancestors, cardinality)); + components.put(MetadataType.HEADER, header.toComponent()); return components; } + + private static List makeList(ByteBuffer[] values) + { + // In most case, l will be the same size than values, but it's possible for it to be smaller + List l = new ArrayList(values.length); + for (int i = 0; i < values.length; i++) + if (values[i] == null) + break; + else + l.add(values[i]); + return l; + } + + public static class MinMaxLongTracker + { + private final long defaultMin; + private final long defaultMax; + + private boolean isSet = false; + private long min; + private long max; + + public MinMaxLongTracker() + { + this(Long.MIN_VALUE, Long.MAX_VALUE); + } + + public MinMaxLongTracker(long defaultMin, long defaultMax) + { + this.defaultMin = defaultMin; + this.defaultMax = defaultMax; + } + + public void update(long value) + { + if (!isSet) + { + min = max = value; + isSet = true; + } + else + { + if (value < min) + min = value; + if (value > max) + max = value; + } + } + + public long min() + { + return isSet ? min : defaultMin; + } + + public long max() + { + return isSet ? max : defaultMax; + } + } + + public static class MinMaxIntTracker + { + private final int defaultMin; + private final int defaultMax; + + private boolean isSet = false; + private int min; + private int max; + + public MinMaxIntTracker() + { + this(Integer.MIN_VALUE, Integer.MAX_VALUE); + } + + public MinMaxIntTracker(int defaultMin, int defaultMax) + { + this.defaultMin = defaultMin; + this.defaultMax = defaultMax; + } + + public void update(int value) + { + if (!isSet) + { + min = max = value; + isSet = true; + } + else + { + if (value < min) + min = value; + if (value > max) + max = value; + } + } + + public int min() + { + return isSet ? min : defaultMin; + } + + public int max() + { + return isSet ? max : defaultMax; + } + } } diff --git a/src/java/org/apache/cassandra/io/sstable/metadata/MetadataSerializer.java b/src/java/org/apache/cassandra/io/sstable/metadata/MetadataSerializer.java index 8a65d8df72..fcdf57ad03 100644 --- a/src/java/org/apache/cassandra/io/sstable/metadata/MetadataSerializer.java +++ b/src/java/org/apache/cassandra/io/sstable/metadata/MetadataSerializer.java @@ -75,7 +75,7 @@ public class MetadataSerializer implements IMetadataSerializer } } - public Map deserialize(Descriptor descriptor, EnumSet types) throws IOException + public Map deserialize( Descriptor descriptor, EnumSet types) throws IOException { Map components; logger.debug("Load metadata for {}", descriptor); diff --git a/src/java/org/apache/cassandra/io/sstable/metadata/MetadataType.java b/src/java/org/apache/cassandra/io/sstable/metadata/MetadataType.java index 9717da1591..875cec43a2 100644 --- a/src/java/org/apache/cassandra/io/sstable/metadata/MetadataType.java +++ b/src/java/org/apache/cassandra/io/sstable/metadata/MetadataType.java @@ -17,6 +17,8 @@ */ package org.apache.cassandra.io.sstable.metadata; +import org.apache.cassandra.db.SerializationHeader; + /** * Defines Metadata component type. */ @@ -27,7 +29,9 @@ public enum MetadataType /** Metadata only used at compaction */ COMPACTION(CompactionMetadata.serializer), /** Metadata always keep in memory */ - STATS(StatsMetadata.serializer); + STATS(StatsMetadata.serializer), + /** Serialization header */ + HEADER((IMetadataComponentSerializer)SerializationHeader.serializer); public final IMetadataComponentSerializer serializer; diff --git a/src/java/org/apache/cassandra/io/sstable/metadata/StatsMetadata.java b/src/java/org/apache/cassandra/io/sstable/metadata/StatsMetadata.java index f2eb1af1f8..809d6b308d 100644 --- a/src/java/org/apache/cassandra/io/sstable/metadata/StatsMetadata.java +++ b/src/java/org/apache/cassandra/io/sstable/metadata/StatsMetadata.java @@ -46,42 +46,57 @@ public class StatsMetadata extends MetadataComponent public final ReplayPosition replayPosition; public final long minTimestamp; public final long maxTimestamp; + public final int minLocalDeletionTime; public final int maxLocalDeletionTime; + public final int minTTL; + public final int maxTTL; public final double compressionRatio; public final StreamingHistogram estimatedTombstoneDropTime; public final int sstableLevel; - public final List maxColumnNames; - public final List minColumnNames; + public final List minClusteringValues; + public final List maxClusteringValues; public final boolean hasLegacyCounterShards; public final long repairedAt; + public final long totalColumnsSet; + public final long totalRows; public StatsMetadata(EstimatedHistogram estimatedRowSize, EstimatedHistogram estimatedColumnCount, ReplayPosition replayPosition, long minTimestamp, long maxTimestamp, + int minLocalDeletionTime, int maxLocalDeletionTime, + int minTTL, + int maxTTL, double compressionRatio, StreamingHistogram estimatedTombstoneDropTime, int sstableLevel, - List minColumnNames, - List maxColumnNames, + List minClusteringValues, + List maxClusteringValues, boolean hasLegacyCounterShards, - long repairedAt) + long repairedAt, + long totalColumnsSet, + long totalRows) { this.estimatedRowSize = estimatedRowSize; this.estimatedColumnCount = estimatedColumnCount; this.replayPosition = replayPosition; this.minTimestamp = minTimestamp; this.maxTimestamp = maxTimestamp; + this.minLocalDeletionTime = minLocalDeletionTime; this.maxLocalDeletionTime = maxLocalDeletionTime; + this.minTTL = minTTL; + this.maxTTL = maxTTL; this.compressionRatio = compressionRatio; this.estimatedTombstoneDropTime = estimatedTombstoneDropTime; this.sstableLevel = sstableLevel; - this.minColumnNames = minColumnNames; - this.maxColumnNames = maxColumnNames; + this.minClusteringValues = minClusteringValues; + this.maxClusteringValues = maxClusteringValues; this.hasLegacyCounterShards = hasLegacyCounterShards; this.repairedAt = repairedAt; + this.totalColumnsSet = totalColumnsSet; + this.totalRows = totalRows; } public MetadataType getType() @@ -120,14 +135,19 @@ public class StatsMetadata extends MetadataComponent replayPosition, minTimestamp, maxTimestamp, + minLocalDeletionTime, maxLocalDeletionTime, + minTTL, + maxTTL, compressionRatio, estimatedTombstoneDropTime, newLevel, - minColumnNames, - maxColumnNames, + minClusteringValues, + maxClusteringValues, hasLegacyCounterShards, - repairedAt); + repairedAt, + totalColumnsSet, + totalRows); } public StatsMetadata mutateRepairedAt(long newRepairedAt) @@ -137,14 +157,19 @@ public class StatsMetadata extends MetadataComponent replayPosition, minTimestamp, maxTimestamp, + minLocalDeletionTime, maxLocalDeletionTime, + minTTL, + maxTTL, compressionRatio, estimatedTombstoneDropTime, sstableLevel, - minColumnNames, - maxColumnNames, + minClusteringValues, + maxClusteringValues, hasLegacyCounterShards, - newRepairedAt); + newRepairedAt, + totalColumnsSet, + totalRows); } @Override @@ -160,14 +185,19 @@ public class StatsMetadata extends MetadataComponent .append(replayPosition, that.replayPosition) .append(minTimestamp, that.minTimestamp) .append(maxTimestamp, that.maxTimestamp) + .append(minLocalDeletionTime, that.minLocalDeletionTime) .append(maxLocalDeletionTime, that.maxLocalDeletionTime) + .append(minTTL, that.minTTL) + .append(maxTTL, that.maxTTL) .append(compressionRatio, that.compressionRatio) .append(estimatedTombstoneDropTime, that.estimatedTombstoneDropTime) .append(sstableLevel, that.sstableLevel) .append(repairedAt, that.repairedAt) - .append(maxColumnNames, that.maxColumnNames) - .append(minColumnNames, that.minColumnNames) + .append(maxClusteringValues, that.maxClusteringValues) + .append(minClusteringValues, that.minClusteringValues) .append(hasLegacyCounterShards, that.hasLegacyCounterShards) + .append(totalColumnsSet, that.totalColumnsSet) + .append(totalRows, that.totalRows) .build(); } @@ -180,14 +210,19 @@ public class StatsMetadata extends MetadataComponent .append(replayPosition) .append(minTimestamp) .append(maxTimestamp) + .append(minLocalDeletionTime) .append(maxLocalDeletionTime) + .append(minTTL) + .append(maxTTL) .append(compressionRatio) .append(estimatedTombstoneDropTime) .append(sstableLevel) .append(repairedAt) - .append(maxColumnNames) - .append(minColumnNames) + .append(maxClusteringValues) + .append(minClusteringValues) .append(hasLegacyCounterShards) + .append(totalColumnsSet) + .append(totalRows) .build(); } @@ -199,18 +234,19 @@ public class StatsMetadata extends MetadataComponent size += EstimatedHistogram.serializer.serializedSize(component.estimatedRowSize, TypeSizes.NATIVE); size += EstimatedHistogram.serializer.serializedSize(component.estimatedColumnCount, TypeSizes.NATIVE); size += ReplayPosition.serializer.serializedSize(component.replayPosition, TypeSizes.NATIVE); - size += 8 + 8 + 4 + 8 + 8; // mix/max timestamp(long), maxLocalDeletionTime(int), compressionRatio(double), repairedAt (long) + size += 8 + 8 + 4 + 4 + 4 + 4 + 8 + 8; // mix/max timestamp(long), min/maxLocalDeletionTime(int), min/max TTL, compressionRatio(double), repairedAt (long) size += StreamingHistogram.serializer.serializedSize(component.estimatedTombstoneDropTime, TypeSizes.NATIVE); size += TypeSizes.NATIVE.sizeof(component.sstableLevel); // min column names size += 4; - for (ByteBuffer columnName : component.minColumnNames) - size += 2 + columnName.remaining(); // with short length + for (ByteBuffer value : component.minClusteringValues) + size += 2 + value.remaining(); // with short length // max column names size += 4; - for (ByteBuffer columnName : component.maxColumnNames) - size += 2 + columnName.remaining(); // with short length + for (ByteBuffer value : component.maxClusteringValues) + size += 2 + value.remaining(); // with short length size += TypeSizes.NATIVE.sizeof(component.hasLegacyCounterShards); + size += 8 + 8; // totalColumnsSet, totalRows return size; } @@ -221,18 +257,24 @@ public class StatsMetadata extends MetadataComponent ReplayPosition.serializer.serialize(component.replayPosition, out); out.writeLong(component.minTimestamp); out.writeLong(component.maxTimestamp); + out.writeInt(component.minLocalDeletionTime); out.writeInt(component.maxLocalDeletionTime); + out.writeInt(component.minTTL); + out.writeInt(component.maxTTL); out.writeDouble(component.compressionRatio); StreamingHistogram.serializer.serialize(component.estimatedTombstoneDropTime, out); out.writeInt(component.sstableLevel); out.writeLong(component.repairedAt); - out.writeInt(component.minColumnNames.size()); - for (ByteBuffer columnName : component.minColumnNames) - ByteBufferUtil.writeWithShortLength(columnName, out); - out.writeInt(component.maxColumnNames.size()); - for (ByteBuffer columnName : component.maxColumnNames) - ByteBufferUtil.writeWithShortLength(columnName, out); + out.writeInt(component.minClusteringValues.size()); + for (ByteBuffer value : component.minClusteringValues) + ByteBufferUtil.writeWithShortLength(value, out); + out.writeInt(component.maxClusteringValues.size()); + for (ByteBuffer value : component.maxClusteringValues) + ByteBufferUtil.writeWithShortLength(value, out); out.writeBoolean(component.hasLegacyCounterShards); + + out.writeLong(component.totalColumnsSet); + out.writeLong(component.totalRows); } public StatsMetadata deserialize(Version version, DataInput in) throws IOException @@ -242,7 +284,11 @@ public class StatsMetadata extends MetadataComponent ReplayPosition replayPosition = ReplayPosition.serializer.deserialize(in); long minTimestamp = in.readLong(); long maxTimestamp = in.readLong(); + // We use MAX_VALUE as that's the default value for "no deletion time" + int minLocalDeletionTime = version.storeRows() ? in.readInt() : Integer.MAX_VALUE; int maxLocalDeletionTime = in.readInt(); + int minTTL = version.storeRows() ? in.readInt() : 0; + int maxTTL = version.storeRows() ? in.readInt() : Integer.MAX_VALUE; double compressionRatio = in.readDouble(); StreamingHistogram tombstoneHistogram = StreamingHistogram.serializer.deserialize(in); int sstableLevel = in.readInt(); @@ -251,32 +297,40 @@ public class StatsMetadata extends MetadataComponent repairedAt = in.readLong(); int colCount = in.readInt(); - List minColumnNames = new ArrayList<>(colCount); + List minClusteringValues = new ArrayList<>(colCount); for (int i = 0; i < colCount; i++) - minColumnNames.add(ByteBufferUtil.readWithShortLength(in)); + minClusteringValues.add(ByteBufferUtil.readWithShortLength(in)); colCount = in.readInt(); - List maxColumnNames = new ArrayList<>(colCount); + List maxClusteringValues = new ArrayList<>(colCount); for (int i = 0; i < colCount; i++) - maxColumnNames.add(ByteBufferUtil.readWithShortLength(in)); + maxClusteringValues.add(ByteBufferUtil.readWithShortLength(in)); boolean hasLegacyCounterShards = true; if (version.tracksLegacyCounterShards()) hasLegacyCounterShards = in.readBoolean(); + long totalColumnsSet = version.storeRows() ? in.readLong() : -1L; + long totalRows = version.storeRows() ? in.readLong() : -1L; + return new StatsMetadata(rowSizes, columnCounts, replayPosition, minTimestamp, maxTimestamp, + minLocalDeletionTime, maxLocalDeletionTime, + minTTL, + maxTTL, compressionRatio, tombstoneHistogram, sstableLevel, - minColumnNames, - maxColumnNames, + minClusteringValues, + maxClusteringValues, hasLegacyCounterShards, - repairedAt); + repairedAt, + totalColumnsSet, + totalRows); } } } diff --git a/src/java/org/apache/cassandra/io/util/DataIntegrityMetadata.java b/src/java/org/apache/cassandra/io/util/DataIntegrityMetadata.java index 4362cee55d..c3a7f984ef 100644 --- a/src/java/org/apache/cassandra/io/util/DataIntegrityMetadata.java +++ b/src/java/org/apache/cassandra/io/util/DataIntegrityMetadata.java @@ -112,10 +112,10 @@ public class DataIntegrityMetadata } catch (Exception e) { + close(); // Attempting to create a FileDigestValidator without a DIGEST file will fail throw new IOException("Corrupted SSTable : " + descriptor.filenameFor(Component.DATA)); } - } // Validate the entire file @@ -133,7 +133,14 @@ public class DataIntegrityMetadata public void close() { - this.digestReader.close(); + try + { + this.digestReader.close(); + } + finally + { + this.dataReader.close(); + } } } diff --git a/src/java/org/apache/cassandra/io/util/FileUtils.java b/src/java/org/apache/cassandra/io/util/FileUtils.java index 35e14199b7..c182e58ab5 100644 --- a/src/java/org/apache/cassandra/io/util/FileUtils.java +++ b/src/java/org/apache/cassandra/io/util/FileUtils.java @@ -227,6 +227,19 @@ public class FileUtils } } + public static void closeQuietly(AutoCloseable c) + { + try + { + if (c != null) + c.close(); + } + catch (Exception e) + { + logger.warn("Failed closing {}", c, e); + } + } + public static void close(Closeable... cs) throws IOException { close(Arrays.asList(cs)); @@ -252,6 +265,22 @@ public class FileUtils throw e; } + public static void closeQuietly(Iterable cs) + { + for (AutoCloseable c : cs) + { + try + { + if (c != null) + c.close(); + } + catch (Exception ex) + { + logger.warn("Failed closing {}", c, ex); + } + } + } + public static String getCanonicalPath(String filename) { try diff --git a/src/java/org/apache/cassandra/net/MessagingService.java b/src/java/org/apache/cassandra/net/MessagingService.java index 83bc337ddb..3f2160f212 100644 --- a/src/java/org/apache/cassandra/net/MessagingService.java +++ b/src/java/org/apache/cassandra/net/MessagingService.java @@ -81,7 +81,8 @@ public final class MessagingService implements MessagingServiceMBean public static final int VERSION_20 = 7; public static final int VERSION_21 = 8; public static final int VERSION_22 = 9; - public static final int current_version = VERSION_22; + public static final int VERSION_30 = 10; + public static final int current_version = VERSION_30; public static final String FAILURE_CALLBACK_PARAM = "CAL_BAC"; public static final byte[] ONE_BYTE = new byte[1]; @@ -104,7 +105,7 @@ public final class MessagingService implements MessagingServiceMBean @Deprecated STREAM_INITIATE_DONE, @Deprecated STREAM_REPLY, @Deprecated STREAM_REQUEST, - RANGE_SLICE, + @Deprecated RANGE_SLICE, @Deprecated BOOTSTRAP_TOKEN, @Deprecated TREE_REQUEST, @Deprecated TREE_RESPONSE, @@ -132,7 +133,7 @@ public final class MessagingService implements MessagingServiceMBean PAXOS_PREPARE, PAXOS_PROPOSE, PAXOS_COMMIT, - PAGED_RANGE, + @Deprecated PAGED_RANGE, // remember to add new verbs at the end, since we serialize by ordinal UNUSED_1, UNUSED_2, @@ -204,8 +205,8 @@ public final class MessagingService implements MessagingServiceMBean put(Verb.MUTATION, Mutation.serializer); put(Verb.READ_REPAIR, Mutation.serializer); put(Verb.READ, ReadCommand.serializer); - put(Verb.RANGE_SLICE, RangeSliceCommand.serializer); - put(Verb.PAGED_RANGE, PagedRangeCommand.serializer); + //put(Verb.RANGE_SLICE, ReadCommand.legacyRangeSliceCommandSerializer); + //put(Verb.PAGED_RANGE, ReadCommand.legacyPagedRangeCommandSerializer); put(Verb.BOOTSTRAP_TOKEN, BootStrapper.StringSerializer.instance); put(Verb.REPAIR_MESSAGE, RepairMessage.serializer); put(Verb.GOSSIP_DIGEST_ACK, GossipDigestAck.serializer); @@ -230,8 +231,8 @@ public final class MessagingService implements MessagingServiceMBean put(Verb.MUTATION, WriteResponse.serializer); put(Verb.READ_REPAIR, WriteResponse.serializer); put(Verb.COUNTER_MUTATION, WriteResponse.serializer); - put(Verb.RANGE_SLICE, RangeSliceReply.serializer); - put(Verb.PAGED_RANGE, RangeSliceReply.serializer); + put(Verb.RANGE_SLICE, ReadResponse.legacyRangeSliceReplySerializer); + put(Verb.PAGED_RANGE, ReadResponse.legacyRangeSliceReplySerializer); put(Verb.READ, ReadResponse.serializer); put(Verb.TRUNCATE, TruncateResponse.serializer); put(Verb.SNAPSHOT, null); diff --git a/src/java/org/apache/cassandra/repair/RepairJob.java b/src/java/org/apache/cassandra/repair/RepairJob.java index 754e26f9ea..ac20428c3d 100644 --- a/src/java/org/apache/cassandra/repair/RepairJob.java +++ b/src/java/org/apache/cassandra/repair/RepairJob.java @@ -180,7 +180,7 @@ public class RepairJob extends AbstractFuture implements Runnable String message = String.format("Requesting merkle trees for %s (to %s)", desc.columnFamily, endpoints); logger.info("[repair #{}] {}", desc.sessionId, message); Tracing.traceRepair(message); - int gcBefore = Keyspace.open(desc.keyspace).getColumnFamilyStore(desc.columnFamily).gcBefore(System.currentTimeMillis()); + int gcBefore = Keyspace.open(desc.keyspace).getColumnFamilyStore(desc.columnFamily).gcBefore(FBUtilities.nowInSeconds()); List> tasks = new ArrayList<>(endpoints.size()); for (InetAddress endpoint : endpoints) { @@ -197,7 +197,7 @@ public class RepairJob extends AbstractFuture implements Runnable */ private ListenableFuture> sendSequentialValidationRequest(Collection endpoints) { - int gcBefore = Keyspace.open(desc.keyspace).getColumnFamilyStore(desc.columnFamily).gcBefore(System.currentTimeMillis()); + int gcBefore = Keyspace.open(desc.keyspace).getColumnFamilyStore(desc.columnFamily).gcBefore(FBUtilities.nowInSeconds()); List> tasks = new ArrayList<>(endpoints.size()); Queue requests = new LinkedList<>(endpoints); @@ -236,7 +236,7 @@ public class RepairJob extends AbstractFuture implements Runnable */ private ListenableFuture> sendDCAwareValidationRequest(Collection endpoints) { - int gcBefore = Keyspace.open(desc.keyspace).getColumnFamilyStore(desc.columnFamily).gcBefore(System.currentTimeMillis()); + int gcBefore = Keyspace.open(desc.keyspace).getColumnFamilyStore(desc.columnFamily).gcBefore(FBUtilities.nowInSeconds()); List> tasks = new ArrayList<>(endpoints.size()); Map> requestsByDatacenter = new HashMap<>(); diff --git a/src/java/org/apache/cassandra/repair/Validator.java b/src/java/org/apache/cassandra/repair/Validator.java index 4db1cfbe92..87d186cd71 100644 --- a/src/java/org/apache/cassandra/repair/Validator.java +++ b/src/java/org/apache/cassandra/repair/Validator.java @@ -31,7 +31,8 @@ import org.apache.cassandra.concurrent.Stage; import org.apache.cassandra.concurrent.StageManager; import org.apache.cassandra.db.ColumnFamilyStore; import org.apache.cassandra.db.DecoratedKey; -import org.apache.cassandra.db.compaction.AbstractCompactedRow; +import org.apache.cassandra.db.rows.UnfilteredRowIterator; +import org.apache.cassandra.db.rows.UnfilteredRowIterators; import org.apache.cassandra.net.MessagingService; import org.apache.cassandra.repair.messages.ValidationComplete; import org.apache.cassandra.tracing.Tracing; @@ -121,18 +122,18 @@ public class Validator implements Runnable * * @param row Row to add hash */ - public void add(AbstractCompactedRow row) + public void add(UnfilteredRowIterator partition) { - assert desc.range.contains(row.key.getToken()) : row.key.getToken() + " is not contained in " + desc.range; - assert lastKey == null || lastKey.compareTo(row.key) < 0 - : "row " + row.key + " received out of order wrt " + lastKey; - lastKey = row.key; + assert desc.range.contains(partition.partitionKey().getToken()) : partition.partitionKey().getToken() + " is not contained in " + desc.range; + assert lastKey == null || lastKey.compareTo(partition.partitionKey()) < 0 + : "partition " + partition.partitionKey() + " received out of order wrt " + lastKey; + lastKey = partition.partitionKey(); if (range == null) range = ranges.next(); // generate new ranges as long as case 1 is true - while (!range.contains(row.key.getToken())) + while (!range.contains(lastKey.getToken())) { // add the empty hash, and move to the next range range.ensureHashInitialised(); @@ -140,7 +141,7 @@ public class Validator implements Runnable } // case 3 must be true: mix in the hashed row - RowHash rowHash = rowHash(row); + RowHash rowHash = rowHash(partition); if (rowHash != null) { range.addHash(rowHash); @@ -186,21 +187,16 @@ public class Validator implements Runnable } - private MerkleTree.RowHash rowHash(AbstractCompactedRow row) + private MerkleTree.RowHash rowHash(UnfilteredRowIterator partition) { validated++; // MerkleTree uses XOR internally, so we want lots of output bits here CountingDigest digest = new CountingDigest(FBUtilities.newMessageDigest("SHA-256")); - row.update(digest); + UnfilteredRowIterators.digest(partition, digest); // only return new hash for merkle tree in case digest was updated - see CASSANDRA-8979 - if (digest.count > 0) - { - return new MerkleTree.RowHash(row.key.getToken(), digest.digest(), digest.count); - } - else - { - return null; - } + return digest.count > 0 + ? new MerkleTree.RowHash(partition.partitionKey().getToken(), digest.digest(), digest.count) + : null; } /** diff --git a/src/java/org/apache/cassandra/schema/LegacySchemaTables.java b/src/java/org/apache/cassandra/schema/LegacySchemaTables.java index b8f6421bc7..1348d12577 100644 --- a/src/java/org/apache/cassandra/schema/LegacySchemaTables.java +++ b/src/java/org/apache/cassandra/schema/LegacySchemaTables.java @@ -23,9 +23,8 @@ import java.security.MessageDigest; import java.security.NoSuchAlgorithmException; import java.util.*; import java.util.concurrent.TimeUnit; +import java.util.function.Function; -import com.google.common.base.Function; -import com.google.common.collect.Iterables; import com.google.common.collect.MapDifference; import com.google.common.collect.Maps; import org.slf4j.Logger; @@ -34,17 +33,14 @@ import org.slf4j.LoggerFactory; import org.apache.cassandra.cache.CachingOptions; import org.apache.cassandra.config.*; import org.apache.cassandra.cql3.*; -import org.apache.cassandra.cql3.functions.*; +import org.apache.cassandra.cql3.functions.AbstractFunction; +import org.apache.cassandra.cql3.functions.FunctionName; +import org.apache.cassandra.cql3.functions.UDFunction; +import org.apache.cassandra.cql3.functions.UDAggregate; import org.apache.cassandra.db.*; -import org.apache.cassandra.db.columniterator.IdentityQueryFilter; -import org.apache.cassandra.db.composites.CellNameType; -import org.apache.cassandra.db.composites.CellNames; -import org.apache.cassandra.db.composites.Composite; -import org.apache.cassandra.db.filter.QueryFilter; -import org.apache.cassandra.db.index.SecondaryIndexManager; +import org.apache.cassandra.db.rows.*; import org.apache.cassandra.db.marshal.*; -import org.apache.cassandra.dht.Range; -import org.apache.cassandra.dht.Token; +import org.apache.cassandra.db.partitions.*; import org.apache.cassandra.exceptions.ConfigurationException; import org.apache.cassandra.exceptions.InvalidRequestException; import org.apache.cassandra.io.compress.CompressionParameters; @@ -52,6 +48,7 @@ import org.apache.cassandra.locator.AbstractReplicationStrategy; import org.apache.cassandra.service.StorageService; import org.apache.cassandra.utils.ByteBufferUtil; import org.apache.cassandra.utils.FBUtilities; +import org.apache.cassandra.utils.concurrent.OpOrder; import static org.apache.cassandra.cql3.QueryProcessor.executeOnceInternal; import static org.apache.cassandra.utils.FBUtilities.fromJsonMap; @@ -100,6 +97,7 @@ public class LegacySchemaTables + "default_time_to_live int," + "default_validator text," + "dropped_columns map," + + "dropped_columns_types map," + "gc_grace_seconds int," + "is_dense boolean," + "key_validator text," @@ -207,29 +205,37 @@ public class LegacySchemaTables public static Collection readSchemaFromSystemTables() { - List serializedSchema = getSchemaPartitionsForTable(KEYSPACES); - - List keyspaces = new ArrayList<>(serializedSchema.size()); - - for (Row partition : serializedSchema) + ReadCommand cmd = getReadCommandForTableSchema(KEYSPACES); + try (ReadOrderGroup orderGroup = cmd.startOrderGroup(); PartitionIterator schema = cmd.executeInternal(orderGroup)) { - if (isEmptySchemaPartition(partition) || isSystemKeyspaceSchemaPartition(partition)) - continue; + List keyspaces = new ArrayList<>(); - keyspaces.add(createKeyspaceFromSchemaPartitions(partition, - readSchemaPartitionForKeyspace(COLUMNFAMILIES, partition.key), - readSchemaPartitionForKeyspace(USERTYPES, partition.key))); + while (schema.hasNext()) + { + try (RowIterator partition = schema.next()) + { + if (isSystemKeyspaceSchemaPartition(partition.partitionKey())) + continue; - // Will be moved away in #6717 - for (UDFunction function : createFunctionsFromFunctionsPartition(readSchemaPartitionForKeyspace(FUNCTIONS, partition.key)).values()) - org.apache.cassandra.cql3.functions.Functions.addOrReplaceFunction(function); + DecoratedKey key = partition.partitionKey(); - // Will be moved away in #6717 - for (UDAggregate aggregate : createAggregatesFromAggregatesPartition(readSchemaPartitionForKeyspace(AGGREGATES, partition.key)).values()) - org.apache.cassandra.cql3.functions.Functions.addOrReplaceFunction(aggregate); + readSchemaPartitionForKeyspaceAndApply(USERTYPES, key, + types -> readSchemaPartitionForKeyspaceAndApply(COLUMNFAMILIES, key, tables -> keyspaces.add(createKeyspaceFromSchemaPartitions(partition, tables, types))) + ); + + // Will be moved away in #6717 + readSchemaPartitionForKeyspaceAndApply(FUNCTIONS, key, + functions -> { createFunctionsFromFunctionsPartition(functions).forEach(function -> org.apache.cassandra.cql3.functions.Functions.addOrReplaceFunction(function)); return null; } + ); + + // Will be moved away in #6717 + readSchemaPartitionForKeyspaceAndApply(AGGREGATES, key, + aggregates -> { createAggregatesFromAggregatesPartition(aggregates).forEach(aggregate -> org.apache.cassandra.cql3.functions.Functions.addOrReplaceFunction(aggregate)); return null; } + ); + } + } + return keyspaces; } - - return keyspaces; } public static void truncateSchemaTables() @@ -262,18 +268,19 @@ public class LegacySchemaTables for (String table : ALL) { - for (Row partition : getSchemaPartitionsForTable(table)) + ReadCommand cmd = getReadCommandForTableSchema(table); + try (ReadOrderGroup orderGroup = cmd.startOrderGroup(); PartitionIterator schema = cmd.executeInternal(orderGroup)) { - if (isEmptySchemaPartition(partition) || isSystemKeyspaceSchemaPartition(partition)) - continue; - - // we want to digest only live columns - ColumnFamilyStore.removeDeletedColumnsOnly(partition.cf, Integer.MAX_VALUE, SecondaryIndexManager.nullUpdater); - partition.cf.purgeTombstones(Integer.MAX_VALUE); - partition.cf.updateDigest(digest); + while (schema.hasNext()) + { + try (RowIterator partition = schema.next()) + { + if (!isSystemKeyspaceSchemaPartition(partition.partitionKey())) + RowIterators.digest(partition, digest); + } + } } } - return UUID.nameUUIDFromBytes(digest.digest()); } @@ -290,14 +297,10 @@ public class LegacySchemaTables * @param schemaTableName The name of the table responsible for part of the schema. * @return low-level schema representation */ - private static List getSchemaPartitionsForTable(String schemaTableName) + private static ReadCommand getReadCommandForTableSchema(String schemaTableName) { - Token minToken = StorageService.getPartitioner().getMinimumToken(); - return getSchemaCFS(schemaTableName).getRangeSlice(new Range(minToken.minKeyBound(), minToken.maxKeyBound()), - null, - new IdentityQueryFilter(), - Integer.MAX_VALUE, - System.currentTimeMillis()); + ColumnFamilyStore cfs = getSchemaCFS(schemaTableName); + return PartitionRangeReadCommand.allDataRead(cfs.metadata, FBUtilities.nowInSeconds()); } public static Collection convertSchemaToMutations() @@ -312,31 +315,45 @@ public class LegacySchemaTables private static void convertSchemaToMutations(Map mutationMap, String schemaTableName) { - for (Row partition : getSchemaPartitionsForTable(schemaTableName)) + ReadCommand cmd = getReadCommandForTableSchema(schemaTableName); + try (ReadOrderGroup orderGroup = cmd.startOrderGroup(); UnfilteredPartitionIterator iter = cmd.executeLocally(orderGroup)) { - if (isSystemKeyspaceSchemaPartition(partition)) - continue; - - Mutation mutation = mutationMap.get(partition.key); - if (mutation == null) + while (iter.hasNext()) { - mutation = new Mutation(SystemKeyspace.NAME, partition.key.getKey()); - mutationMap.put(partition.key, mutation); - } + try (UnfilteredRowIterator partition = iter.next()) + { + if (isSystemKeyspaceSchemaPartition(partition.partitionKey())) + continue; - mutation.add(partition.cf); + DecoratedKey key = partition.partitionKey(); + Mutation mutation = mutationMap.get(key); + if (mutation == null) + { + mutation = new Mutation(SystemKeyspace.NAME, key); + mutationMap.put(key, mutation); + } + + mutation.add(UnfilteredRowIterators.toUpdate(partition)); + } + } } } - private static Map readSchemaForKeyspaces(String schemaTableName, Set keyspaceNames) + private static Map readSchemaForKeyspaces(String schemaTableName, Set keyspaceNames) { - Map schema = new HashMap<>(); + Map schema = new HashMap<>(); for (String keyspaceName : keyspaceNames) { - Row schemaEntity = readSchemaPartitionForKeyspace(schemaTableName, keyspaceName); - if (schemaEntity.cf != null) - schema.put(schemaEntity.key, schemaEntity.cf); + // We don't to return the RowIterator directly because we should guarantee that this iterator + // will be closed, and putting it in a Map make that harder/more awkward. + readSchemaPartitionForKeyspaceAndApply(schemaTableName, keyspaceName, + partition -> { + if (!partition.isEmpty()) + schema.put(partition.partitionKey(), FilteredPartition.create(partition)); + return null; + } + ); } return schema; @@ -347,35 +364,46 @@ public class LegacySchemaTables return AsciiType.instance.fromString(ksName); } - private static Row readSchemaPartitionForKeyspace(String schemaTableName, String keyspaceName) + private static DecoratedKey getSchemaKSDecoratedKey(String ksName) { - DecoratedKey keyspaceKey = StorageService.getPartitioner().decorateKey(getSchemaKSKey(keyspaceName)); - return readSchemaPartitionForKeyspace(schemaTableName, keyspaceKey); + return StorageService.getPartitioner().decorateKey(getSchemaKSKey(ksName)); } - private static Row readSchemaPartitionForKeyspace(String schemaTableName, DecoratedKey keyspaceKey) + private static T readSchemaPartitionForKeyspaceAndApply(String schemaTableName, String keyspaceName, Function fct) { - QueryFilter filter = QueryFilter.getIdentityFilter(keyspaceKey, schemaTableName, System.currentTimeMillis()); - return new Row(keyspaceKey, getSchemaCFS(schemaTableName).getColumnFamily(filter)); + return readSchemaPartitionForKeyspaceAndApply(schemaTableName, getSchemaKSDecoratedKey(keyspaceName), fct); } - private static Row readSchemaPartitionForTable(String schemaTableName, String keyspaceName, String tableName) + private static T readSchemaPartitionForKeyspaceAndApply(String schemaTableName, DecoratedKey keyspaceKey, Function fct) { - DecoratedKey key = StorageService.getPartitioner().decorateKey(getSchemaKSKey(keyspaceName)); ColumnFamilyStore store = getSchemaCFS(schemaTableName); - Composite prefix = store.getComparator().make(tableName); - ColumnFamily cells = store.getColumnFamily(key, prefix, prefix.end(), false, Integer.MAX_VALUE, System.currentTimeMillis()); - return new Row(key, cells); + int nowInSec = FBUtilities.nowInSeconds(); + try (OpOrder.Group op = store.readOrdering.start(); + RowIterator partition = UnfilteredRowIterators.filter(SinglePartitionReadCommand.fullPartitionRead(store.metadata, nowInSec, keyspaceKey) + .queryMemtableAndDisk(store, op), nowInSec)) + { + return fct.apply(partition); + } } - private static boolean isEmptySchemaPartition(Row partition) + private static T readSchemaPartitionForTableAndApply(String schemaTableName, String keyspaceName, String tableName, Function fct) { - return partition.cf == null || (partition.cf.isMarkedForDelete() && !partition.cf.hasColumns()); + ColumnFamilyStore store = getSchemaCFS(schemaTableName); + + ClusteringComparator comparator = store.metadata.comparator; + Slices slices = Slices.with(comparator, Slice.make(comparator, tableName)); + int nowInSec = FBUtilities.nowInSeconds(); + try (OpOrder.Group op = store.readOrdering.start(); + RowIterator partition = UnfilteredRowIterators.filter(SinglePartitionSliceCommand.create(store.metadata, nowInSec, getSchemaKSDecoratedKey(keyspaceName), slices) + .queryMemtableAndDisk(store, op), nowInSec)) + { + return fct.apply(partition); + } } - private static boolean isSystemKeyspaceSchemaPartition(Row partition) + private static boolean isSystemKeyspaceSchemaPartition(DecoratedKey partitionKey) { - return getSchemaKSKey(SystemKeyspace.NAME).equals(partition.key.getKey()); + return getSchemaKSKey(SystemKeyspace.NAME).equals(partitionKey.getKey()); } /** @@ -398,14 +426,14 @@ public class LegacySchemaTables // compare before/after schemas of the affected keyspaces only Set keyspaces = new HashSet<>(mutations.size()); for (Mutation mutation : mutations) - keyspaces.add(ByteBufferUtil.string(mutation.key())); + keyspaces.add(ByteBufferUtil.string(mutation.key().getKey())); // current state of the schema - Map oldKeyspaces = readSchemaForKeyspaces(KEYSPACES, keyspaces); - Map oldColumnFamilies = readSchemaForKeyspaces(COLUMNFAMILIES, keyspaces); - Map oldTypes = readSchemaForKeyspaces(USERTYPES, keyspaces); - Map oldFunctions = readSchemaForKeyspaces(FUNCTIONS, keyspaces); - Map oldAggregates = readSchemaForKeyspaces(AGGREGATES, keyspaces); + Map oldKeyspaces = readSchemaForKeyspaces(KEYSPACES, keyspaces); + Map oldColumnFamilies = readSchemaForKeyspaces(COLUMNFAMILIES, keyspaces); + Map oldTypes = readSchemaForKeyspaces(USERTYPES, keyspaces); + Map oldFunctions = readSchemaForKeyspaces(FUNCTIONS, keyspaces); + Map oldAggregates = readSchemaForKeyspaces(AGGREGATES, keyspaces); for (Mutation mutation : mutations) mutation.apply(); @@ -414,11 +442,11 @@ public class LegacySchemaTables flushSchemaTables(); // with new data applied - Map newKeyspaces = readSchemaForKeyspaces(KEYSPACES, keyspaces); - Map newColumnFamilies = readSchemaForKeyspaces(COLUMNFAMILIES, keyspaces); - Map newTypes = readSchemaForKeyspaces(USERTYPES, keyspaces); - Map newFunctions = readSchemaForKeyspaces(FUNCTIONS, keyspaces); - Map newAggregates = readSchemaForKeyspaces(AGGREGATES, keyspaces); + Map newKeyspaces = readSchemaForKeyspaces(KEYSPACES, keyspaces); + Map newColumnFamilies = readSchemaForKeyspaces(COLUMNFAMILIES, keyspaces); + Map newTypes = readSchemaForKeyspaces(USERTYPES, keyspaces); + Map newFunctions = readSchemaForKeyspaces(FUNCTIONS, keyspaces); + Map newAggregates = readSchemaForKeyspaces(AGGREGATES, keyspaces); Set keyspacesToDrop = mergeKeyspaces(oldKeyspaces, newKeyspaces); mergeTables(oldColumnFamilies, newColumnFamilies); @@ -431,263 +459,187 @@ public class LegacySchemaTables Schema.instance.dropKeyspace(keyspaceToDrop); } - private static Set mergeKeyspaces(Map before, Map after) + private static Set mergeKeyspaces(Map before, Map after) { - List created = new ArrayList<>(); - List altered = new ArrayList<>(); - Set dropped = new HashSet<>(); - - /* - * - we don't care about entriesOnlyOnLeft() or entriesInCommon(), because only the changes are of interest to us - * - of all entriesOnlyOnRight(), we only care about ones that have live columns; it's possible to have a ColumnFamily - * there that only has the top-level deletion, if: - * a) a pushed DROP KEYSPACE change for a keyspace hadn't ever made it to this node in the first place - * b) a pulled dropped keyspace that got dropped before it could find a way to this node - * - of entriesDiffering(), we don't care about the scenario where both pre and post-values have zero live columns: - * that means that a keyspace had been recreated and dropped, and the recreated keyspace had never found a way - * to this node - */ - MapDifference diff = Maps.difference(before, after); - - for (Map.Entry entry : diff.entriesOnlyOnRight().entrySet()) - if (entry.getValue().hasColumns()) - created.add(new Row(entry.getKey(), entry.getValue())); - - for (Map.Entry> entry : diff.entriesDiffering().entrySet()) + for (FilteredPartition newPartition : after.values()) { - String keyspaceName = AsciiType.instance.compose(entry.getKey().getKey()); - - ColumnFamily pre = entry.getValue().leftValue(); - ColumnFamily post = entry.getValue().rightValue(); - - if (pre.hasColumns() && post.hasColumns()) - altered.add(keyspaceName); - else if (pre.hasColumns()) - dropped.add(keyspaceName); - else if (post.hasColumns()) // a (re)created keyspace - created.add(new Row(entry.getKey(), post)); + FilteredPartition oldPartition = before.remove(newPartition.partitionKey()); + if (oldPartition == null || oldPartition.isEmpty()) + { + Schema.instance.addKeyspace(createKeyspaceFromSchemaPartition(newPartition.rowIterator())); + } + else + { + String name = AsciiType.instance.compose(newPartition.partitionKey().getKey()); + Schema.instance.updateKeyspace(name); + } } - for (Row row : created) - Schema.instance.addKeyspace(createKeyspaceFromSchemaPartition(row)); - for (String name : altered) - Schema.instance.updateKeyspace(name); - return dropped; + // What's remain in old is those keyspace that are not in updated, i.e. the dropped ones. + return asKeyspaceNamesSet(before.keySet()); } - // see the comments for mergeKeyspaces() - private static void mergeTables(Map before, Map after) + private static Set asKeyspaceNamesSet(Set keys) { - List created = new ArrayList<>(); - List altered = new ArrayList<>(); - List dropped = new ArrayList<>(); - - MapDifference diff = Maps.difference(before, after); - - for (Map.Entry entry : diff.entriesOnlyOnRight().entrySet()) - if (entry.getValue().hasColumns()) - created.addAll(createTablesFromTablesPartition(new Row(entry.getKey(), entry.getValue())).values()); - - for (Map.Entry> entry : diff.entriesDiffering().entrySet()) - { - String keyspaceName = AsciiType.instance.compose(entry.getKey().getKey()); - - ColumnFamily pre = entry.getValue().leftValue(); - ColumnFamily post = entry.getValue().rightValue(); - - if (pre.hasColumns() && post.hasColumns()) - { - MapDifference delta = - Maps.difference(Schema.instance.getKSMetaData(keyspaceName).cfMetaData(), - createTablesFromTablesPartition(new Row(entry.getKey(), post))); - - dropped.addAll(delta.entriesOnlyOnLeft().values()); - created.addAll(delta.entriesOnlyOnRight().values()); - Iterables.addAll(altered, Iterables.transform(delta.entriesDiffering().values(), new Function, CFMetaData>() - { - public CFMetaData apply(MapDifference.ValueDifference pair) - { - return pair.rightValue(); - } - })); - } - else if (pre.hasColumns()) - { - dropped.addAll(Schema.instance.getKSMetaData(keyspaceName).cfMetaData().values()); - } - else if (post.hasColumns()) - { - created.addAll(createTablesFromTablesPartition(new Row(entry.getKey(), post)).values()); - } - } - - for (CFMetaData cfm : created) - Schema.instance.addTable(cfm); - for (CFMetaData cfm : altered) - Schema.instance.updateTable(cfm.ksName, cfm.cfName); - for (CFMetaData cfm : dropped) - Schema.instance.dropTable(cfm.ksName, cfm.cfName); + Set names = new HashSet(keys.size()); + for (DecoratedKey key : keys) + names.add(AsciiType.instance.compose(key.getKey())); + return names; } - // see the comments for mergeKeyspaces() - private static void mergeTypes(Map before, Map after) + private static void mergeTables(Map before, Map after) { - List created = new ArrayList<>(); - List altered = new ArrayList<>(); - List dropped = new ArrayList<>(); - - MapDifference diff = Maps.difference(before, after); - - // New keyspace with types - for (Map.Entry entry : diff.entriesOnlyOnRight().entrySet()) - if (entry.getValue().hasColumns()) - created.addAll(createTypesFromPartition(new Row(entry.getKey(), entry.getValue())).values()); - - for (Map.Entry> entry : diff.entriesDiffering().entrySet()) + diffSchema(before, after, new Differ() { - String keyspaceName = AsciiType.instance.compose(entry.getKey().getKey()); - - ColumnFamily pre = entry.getValue().leftValue(); - ColumnFamily post = entry.getValue().rightValue(); - - if (pre.hasColumns() && post.hasColumns()) + public void onDropped(UntypedResultSet.Row oldRow) { - MapDifference delta = - Maps.difference(Schema.instance.getKSMetaData(keyspaceName).userTypes.getAllTypes(), - createTypesFromPartition(new Row(entry.getKey(), post))); + Schema.instance.dropTable(oldRow.getString("keyspace_name"), oldRow.getString("columnfamily_name")); + } - dropped.addAll(delta.entriesOnlyOnLeft().values()); - created.addAll(delta.entriesOnlyOnRight().values()); - Iterables.addAll(altered, Iterables.transform(delta.entriesDiffering().values(), new Function, UserType>() - { - public UserType apply(MapDifference.ValueDifference pair) - { - return pair.rightValue(); - } - })); - } - else if (pre.hasColumns()) + public void onAdded(UntypedResultSet.Row newRow) { - dropped.addAll(Schema.instance.getKSMetaData(keyspaceName).userTypes.getAllTypes().values()); + Schema.instance.addTable(createTableFromTableRow(newRow)); } - else if (post.hasColumns()) - { - created.addAll(createTypesFromPartition(new Row(entry.getKey(), post)).values()); - } - } - for (UserType type : created) - Schema.instance.addType(type); - for (UserType type : altered) - Schema.instance.updateType(type); - for (UserType type : dropped) - Schema.instance.dropType(type); + public void onUpdated(UntypedResultSet.Row oldRow, UntypedResultSet.Row newRow) + { + Schema.instance.updateTable(newRow.getString("keyspace_name"), newRow.getString("columnfamily_name")); + } + }); } - // see the comments for mergeKeyspaces() - private static void mergeFunctions(Map before, Map after) + private static void mergeTypes(Map before, Map after) { - List created = new ArrayList<>(); - List altered = new ArrayList<>(); - List dropped = new ArrayList<>(); - - MapDifference diff = Maps.difference(before, after); - - // New keyspace with functions - for (Map.Entry entry : diff.entriesOnlyOnRight().entrySet()) - if (entry.getValue().hasColumns()) - created.addAll(createFunctionsFromFunctionsPartition(new Row(entry.getKey(), entry.getValue())).values()); - - for (Map.Entry> entry : diff.entriesDiffering().entrySet()) + diffSchema(before, after, new Differ() { - ColumnFamily pre = entry.getValue().leftValue(); - ColumnFamily post = entry.getValue().rightValue(); - - if (pre.hasColumns() && post.hasColumns()) + public void onDropped(UntypedResultSet.Row oldRow) { - MapDifference delta = - Maps.difference(createFunctionsFromFunctionsPartition(new Row(entry.getKey(), pre)), - createFunctionsFromFunctionsPartition(new Row(entry.getKey(), post))); + Schema.instance.dropType(createTypeFromRow(oldRow)); + } - dropped.addAll(delta.entriesOnlyOnLeft().values()); - created.addAll(delta.entriesOnlyOnRight().values()); - Iterables.addAll(altered, Iterables.transform(delta.entriesDiffering().values(), new Function, UDFunction>() - { - public UDFunction apply(MapDifference.ValueDifference pair) - { - return pair.rightValue(); - } - })); - } - else if (pre.hasColumns()) + public void onAdded(UntypedResultSet.Row newRow) { - dropped.addAll(createFunctionsFromFunctionsPartition(new Row(entry.getKey(), pre)).values()); + Schema.instance.addType(createTypeFromRow(newRow)); } - else if (post.hasColumns()) - { - created.addAll(createFunctionsFromFunctionsPartition(new Row(entry.getKey(), post)).values()); - } - } - for (UDFunction udf : created) - Schema.instance.addFunction(udf); - for (UDFunction udf : altered) - Schema.instance.updateFunction(udf); - for (UDFunction udf : dropped) - Schema.instance.dropFunction(udf); + public void onUpdated(UntypedResultSet.Row oldRow, UntypedResultSet.Row newRow) + { + Schema.instance.updateType(createTypeFromRow(newRow)); + } + }); } - // see the comments for mergeKeyspaces() - private static void mergeAggregates(Map before, Map after) + private static void mergeFunctions(Map before, Map after) { - List created = new ArrayList<>(); - List altered = new ArrayList<>(); - List dropped = new ArrayList<>(); - - MapDifference diff = Maps.difference(before, after); - - // New keyspace with functions - for (Map.Entry entry : diff.entriesOnlyOnRight().entrySet()) - if (entry.getValue().hasColumns()) - created.addAll(createAggregatesFromAggregatesPartition(new Row(entry.getKey(), entry.getValue())).values()); - - for (Map.Entry> entry : diff.entriesDiffering().entrySet()) + diffSchema(before, after, new Differ() { - ColumnFamily pre = entry.getValue().leftValue(); - ColumnFamily post = entry.getValue().rightValue(); - - if (pre.hasColumns() && post.hasColumns()) + public void onDropped(UntypedResultSet.Row oldRow) { - MapDifference delta = - Maps.difference(createAggregatesFromAggregatesPartition(new Row(entry.getKey(), pre)), - createAggregatesFromAggregatesPartition(new Row(entry.getKey(), post))); + Schema.instance.dropFunction(createFunctionFromFunctionRow(oldRow)); + } - dropped.addAll(delta.entriesOnlyOnLeft().values()); - created.addAll(delta.entriesOnlyOnRight().values()); - Iterables.addAll(altered, Iterables.transform(delta.entriesDiffering().values(), new Function, UDAggregate>() + public void onAdded(UntypedResultSet.Row newRow) + { + Schema.instance.addFunction(createFunctionFromFunctionRow(newRow)); + } + + public void onUpdated(UntypedResultSet.Row oldRow, UntypedResultSet.Row newRow) + { + Schema.instance.updateFunction(createFunctionFromFunctionRow(newRow)); + } + }); + } + + private static void mergeAggregates(Map before, Map after) + { + diffSchema(before, after, new Differ() + { + public void onDropped(UntypedResultSet.Row oldRow) + { + Schema.instance.dropAggregate(createAggregateFromAggregateRow(oldRow)); + } + + public void onAdded(UntypedResultSet.Row newRow) + { + Schema.instance.addAggregate(createAggregateFromAggregateRow(newRow)); + } + + public void onUpdated(UntypedResultSet.Row oldRow, UntypedResultSet.Row newRow) + { + Schema.instance.updateAggregate(createAggregateFromAggregateRow(newRow)); + } + }); + } + + public interface Differ + { + public void onDropped(UntypedResultSet.Row oldRow); + public void onAdded(UntypedResultSet.Row newRow); + public void onUpdated(UntypedResultSet.Row oldRow, UntypedResultSet.Row newRow); + } + + private static void diffSchema(Map before, Map after, Differ differ) + { + for (FilteredPartition newPartition : after.values()) + { + CFMetaData metadata = newPartition.metadata(); + DecoratedKey key = newPartition.partitionKey(); + + FilteredPartition oldPartition = before.remove(key); + + if (oldPartition == null || oldPartition.isEmpty()) + { + // Means everything is to be added + for (Row row : newPartition) + differ.onAdded(UntypedResultSet.Row.fromInternalRow(metadata, key, row)); + continue; + } + + Iterator oldIter = oldPartition.iterator(); + Iterator newIter = newPartition.iterator(); + + Row oldRow = oldIter.hasNext() ? oldIter.next() : null; + Row newRow = newIter.hasNext() ? newIter.next() : null; + while (oldRow != null && newRow != null) + { + int cmp = metadata.comparator.compare(oldRow.clustering(), newRow.clustering()); + if (cmp < 0) { - public UDAggregate apply(MapDifference.ValueDifference pair) - { - return pair.rightValue(); - } - })); + differ.onDropped(UntypedResultSet.Row.fromInternalRow(metadata, key, oldRow)); + oldRow = oldIter.hasNext() ? oldIter.next() : null; + } + else if (cmp > 0) + { + + differ.onAdded(UntypedResultSet.Row.fromInternalRow(metadata, key, newRow)); + newRow = newIter.hasNext() ? newIter.next() : null; + } + else + { + if (!oldRow.equals(newRow)) + differ.onUpdated(UntypedResultSet.Row.fromInternalRow(metadata, key, oldRow), UntypedResultSet.Row.fromInternalRow(metadata, key, newRow)); + + oldRow = oldIter.hasNext() ? oldIter.next() : null; + newRow = newIter.hasNext() ? newIter.next() : null; + } } - else if (pre.hasColumns()) + + while (oldRow != null) { - dropped.addAll(createAggregatesFromAggregatesPartition(new Row(entry.getKey(), pre)).values()); + differ.onDropped(UntypedResultSet.Row.fromInternalRow(metadata, key, oldRow)); + oldRow = oldIter.hasNext() ? oldIter.next() : null; } - else if (post.hasColumns()) + while (newRow != null) { - created.addAll(createAggregatesFromAggregatesPartition(new Row(entry.getKey(), post)).values()); + differ.onAdded(UntypedResultSet.Row.fromInternalRow(metadata, key, newRow)); + newRow = newIter.hasNext() ? newIter.next() : null; } } - for (UDAggregate udf : created) - Schema.instance.addAggregate(udf); - for (UDAggregate udf : altered) - Schema.instance.updateAggregate(udf); - for (UDAggregate udf : dropped) - Schema.instance.dropAggregate(udf); + // What remains is those keys that were only in before. + for (FilteredPartition partition : before.values()) + for (Row row : partition) + differ.onDropped(UntypedResultSet.Row.fromInternalRow(partition.metadata(), partition.partitionKey(), row)); } /* @@ -701,14 +653,15 @@ public class LegacySchemaTables private static Mutation makeCreateKeyspaceMutation(KSMetaData keyspace, long timestamp, boolean withTablesAndTypesAndFunctions) { - Mutation mutation = new Mutation(SystemKeyspace.NAME, getSchemaKSKey(keyspace.name)); - ColumnFamily cells = mutation.addOrGet(Keyspaces); - CFRowAdder adder = new CFRowAdder(cells, Keyspaces.comparator.builder().build(), timestamp); + // Note that because Keyspaces is a COMPACT TABLE, we're really only setting static columns internally and shouldn't set any clustering. + RowUpdateBuilder adder = new RowUpdateBuilder(Keyspaces, timestamp, keyspace.name); adder.add("durable_writes", keyspace.durableWrites); adder.add("strategy_class", keyspace.strategyClass.getName()); adder.add("strategy_options", json(keyspace.strategyOptions)); + Mutation mutation = adder.build(); + if (withTablesAndTypesAndFunctions) { for (UserType type : keyspace.userTypes.getAllTypes().values()) @@ -723,36 +676,39 @@ public class LegacySchemaTables public static Mutation makeDropKeyspaceMutation(KSMetaData keyspace, long timestamp) { - Mutation mutation = new Mutation(SystemKeyspace.NAME, getSchemaKSKey(keyspace.name)); - for (String schemaTable : ALL) - mutation.delete(schemaTable, timestamp); - mutation.delete(SystemKeyspace.BUILT_INDEXES, timestamp); + int nowInSec = FBUtilities.nowInSeconds(); + Mutation mutation = new Mutation(SystemKeyspace.NAME, getSchemaKSDecoratedKey(keyspace.name)); + for (CFMetaData schemaTable : All) + mutation.add(PartitionUpdate.fullPartitionDelete(schemaTable, mutation.key(), timestamp, nowInSec)); + mutation.add(PartitionUpdate.fullPartitionDelete(SystemKeyspace.BuiltIndexes, mutation.key(), timestamp, nowInSec)); return mutation; } - private static KSMetaData createKeyspaceFromSchemaPartitions(Row serializedKeyspace, Row serializedTables, Row serializedTypes) + private static KSMetaData createKeyspaceFromSchemaPartitions(RowIterator serializedKeyspace, RowIterator serializedTables, RowIterator serializedTypes) { - Collection tables = createTablesFromTablesPartition(serializedTables).values(); + Collection tables = createTablesFromTablesPartition(serializedTables); UTMetaData types = new UTMetaData(createTypesFromPartition(serializedTypes)); return createKeyspaceFromSchemaPartition(serializedKeyspace).cloneWith(tables, types); } public static KSMetaData createKeyspaceFromName(String keyspace) { - Row partition = readSchemaPartitionForKeyspace(KEYSPACES, keyspace); + return readSchemaPartitionForKeyspaceAndApply(KEYSPACES, keyspace, partition -> + { + if (partition.isEmpty()) + throw new RuntimeException(String.format("%s not found in the schema definitions keyspaceName (%s).", keyspace, KEYSPACES)); - if (isEmptySchemaPartition(partition)) - throw new RuntimeException(String.format("%s not found in the schema definitions keyspaceName (%s).", keyspace, KEYSPACES)); - - return createKeyspaceFromSchemaPartition(partition); + return createKeyspaceFromSchemaPartition(partition); + }); } + /** * Deserialize only Keyspace attributes without nested tables or types * * @param partition Keyspace attributes in serialized form */ - private static KSMetaData createKeyspaceFromSchemaPartition(Row partition) + private static KSMetaData createKeyspaceFromSchemaPartition(RowIterator partition) { String query = String.format("SELECT * FROM %s.%s", SystemKeyspace.NAME, KEYSPACES); UntypedResultSet.Row row = QueryProcessor.resultify(query, partition).one(); @@ -776,10 +732,8 @@ public class LegacySchemaTables private static void addTypeToSchemaMutation(UserType type, long timestamp, Mutation mutation) { - ColumnFamily cells = mutation.addOrGet(Usertypes); - - Composite prefix = Usertypes.comparator.make(type.name); - CFRowAdder adder = new CFRowAdder(cells, prefix, timestamp); + RowUpdateBuilder adder = new RowUpdateBuilder(Usertypes, timestamp, mutation) + .clustering(type.name); adder.resetCollection("field_names"); adder.resetCollection("field_types"); @@ -789,23 +743,18 @@ public class LegacySchemaTables adder.addListEntry("field_names", type.fieldName(i)); adder.addListEntry("field_types", type.fieldType(i).toString()); } + + adder.build(); } public static Mutation dropTypeFromSchemaMutation(KSMetaData keyspace, UserType type, long timestamp) { // Include the serialized keyspace in case the target node missed a CREATE KEYSPACE migration (see CASSANDRA-5631). Mutation mutation = makeCreateKeyspaceMutation(keyspace, timestamp, false); - - ColumnFamily cells = mutation.addOrGet(Usertypes); - int ldt = (int) (System.currentTimeMillis() / 1000); - - Composite prefix = Usertypes.comparator.make(type.name); - cells.addAtom(new RangeTombstone(prefix, prefix.end(), timestamp, ldt)); - - return mutation; + return RowUpdateBuilder.deleteRow(Usertypes, timestamp, mutation, type.name); } - private static Map createTypesFromPartition(Row partition) + private static Map createTypesFromPartition(RowIterator partition) { String query = String.format("SELECT * FROM %s.%s", SystemKeyspace.NAME, USERTYPES); Map types = new HashMap<>(); @@ -851,12 +800,11 @@ public class LegacySchemaTables { // For property that can be null (and can be changed), we insert tombstones, to make sure // we don't keep a property the user has removed - ColumnFamily cells = mutation.addOrGet(Columnfamilies); - Composite prefix = Columnfamilies.comparator.make(table.cfName); - CFRowAdder adder = new CFRowAdder(cells, prefix, timestamp); + RowUpdateBuilder adder = new RowUpdateBuilder(Columnfamilies, timestamp, mutation) + .clustering(table.cfName); adder.add("cf_id", table.cfId); - adder.add("type", table.cfType.toString()); + adder.add("type", table.isSuper() ? "Super" : "Standard"); if (table.isSuper()) { @@ -864,11 +812,11 @@ public class LegacySchemaTables // we won't know at deserialization if the subcomparator should be taken into account // TODO: we should implement an on-start migration if we want to get rid of that. adder.add("comparator", table.comparator.subtype(0).toString()); - adder.add("subcomparator", table.comparator.subtype(1).toString()); + adder.add("subcomparator", ((MapType)table.compactValueColumn().type).getKeysType().toString()); } else { - adder.add("comparator", table.comparator.toString()); + adder.add("comparator", LegacyLayout.makeLegacyComparator(table).toString()); } adder.add("bloom_filter_fp_chance", table.getBloomFilterFpChance()); @@ -878,7 +826,6 @@ public class LegacySchemaTables adder.add("compaction_strategy_options", json(table.compactionStrategyOptions)); adder.add("compression_parameters", json(table.compressionParameters.asThriftOptions())); adder.add("default_time_to_live", table.getDefaultTimeToLive()); - adder.add("default_validator", table.getDefaultValidator().toString()); adder.add("gc_grace_seconds", table.getGcGraceSeconds()); adder.add("key_validator", table.getKeyValidator().toString()); adder.add("local_read_repair_chance", table.getDcLocalReadRepairChance()); @@ -890,10 +837,18 @@ public class LegacySchemaTables adder.add("read_repair_chance", table.getReadRepairChance()); adder.add("speculative_retry", table.getSpeculativeRetry().toString()); - for (Map.Entry entry : table.getDroppedColumns().entrySet()) - adder.addMapEntry("dropped_columns", entry.getKey().toString(), entry.getValue()); + for (Map.Entry entry : table.getDroppedColumns().entrySet()) + { + String name = entry.getKey().toString(); + CFMetaData.DroppedColumn column = entry.getValue(); + adder.addMapEntry("dropped_columns", name, column.droppedTime); + if (column.type != null) + adder.addMapEntry("dropped_columns_types", name, column.type.toString()); + } - adder.add("is_dense", table.getIsDense()); + adder.add("is_dense", table.isDense()); + + adder.add("default_validator", table.makeLegacyDefaultValidator().toString()); if (withColumnsAndTriggers) { @@ -903,6 +858,8 @@ public class LegacySchemaTables for (TriggerDefinition trigger : table.getTriggers().values()) addTriggerToSchemaMutation(table, trigger, timestamp, mutation); } + + adder.build(); } public static Mutation makeUpdateTableMutation(KSMetaData keyspace, @@ -955,11 +912,7 @@ public class LegacySchemaTables // Include the serialized keyspace in case the target node missed a CREATE KEYSPACE migration (see CASSANDRA-5631). Mutation mutation = makeCreateKeyspaceMutation(keyspace, timestamp, false); - ColumnFamily cells = mutation.addOrGet(Columnfamilies); - int ldt = (int) (System.currentTimeMillis() / 1000); - - Composite prefix = Columnfamilies.comparator.make(table.cfName); - cells.addAtom(new RangeTombstone(prefix, prefix.end(), timestamp, ldt)); + RowUpdateBuilder.deleteRow(Columnfamilies, timestamp, mutation, table.cfName); for (ColumnDefinition column : table.allColumns()) dropColumnFromSchemaMutation(table, column, timestamp, mutation); @@ -968,21 +921,21 @@ public class LegacySchemaTables dropTriggerFromSchemaMutation(table, trigger, timestamp, mutation); // TODO: get rid of in #6717 - ColumnFamily indexCells = mutation.addOrGet(SystemKeyspace.BuiltIndexes); for (String indexName : Keyspace.open(keyspace.name).getColumnFamilyStore(table.cfName).getBuiltIndexes()) - indexCells.addTombstone(indexCells.getComparator().makeCellName(indexName), ldt, timestamp); + RowUpdateBuilder.deleteRow(SystemKeyspace.BuiltIndexes, timestamp, mutation, indexName); return mutation; } public static CFMetaData createTableFromName(String keyspace, String table) { - Row partition = readSchemaPartitionForTable(COLUMNFAMILIES, keyspace, table); + return readSchemaPartitionForTableAndApply(COLUMNFAMILIES, keyspace, table, partition -> + { + if (partition.isEmpty()) + throw new RuntimeException(String.format("%s:%s not found in the schema definitions keyspace.", keyspace, table)); - if (isEmptySchemaPartition(partition)) - throw new RuntimeException(String.format("%s:%s not found in the schema definitions keyspace.", keyspace, table)); - - return createTableFromTablePartition(partition); + return createTableFromTablePartition(partition); + }); } /** @@ -990,37 +943,34 @@ public class LegacySchemaTables * * @return map containing name of the table and its metadata for faster lookup */ - private static Map createTablesFromTablesPartition(Row partition) + private static Collection createTablesFromTablesPartition(RowIterator partition) { - if (partition.cf == null) - return Collections.emptyMap(); + if (partition.isEmpty()) + return Collections.emptyList(); String query = String.format("SELECT * FROM %s.%s", SystemKeyspace.NAME, COLUMNFAMILIES); - Map tables = new HashMap<>(); + List tables = new ArrayList<>(); for (UntypedResultSet.Row row : QueryProcessor.resultify(query, partition)) - { - CFMetaData cfm = createTableFromTableRow(row); - tables.put(cfm.cfName, cfm); - } + tables.add(createTableFromTableRow(row)); return tables; } - public static CFMetaData createTableFromTablePartitionAndColumnsPartition(Row serializedTable, Row serializedColumns) + public static CFMetaData createTableFromTablePartitionAndColumnsPartition(RowIterator serializedTable, RowIterator serializedColumns) { String query = String.format("SELECT * FROM %s.%s", SystemKeyspace.NAME, COLUMNFAMILIES); return createTableFromTableRowAndColumnsPartition(QueryProcessor.resultify(query, serializedTable).one(), serializedColumns); } - private static CFMetaData createTableFromTableRowAndColumnsPartition(UntypedResultSet.Row tableRow, Row serializedColumns) + private static CFMetaData createTableFromTableRowAndColumnsPartition(UntypedResultSet.Row tableRow, RowIterator serializedColumns) { String query = String.format("SELECT * FROM %s.%s", SystemKeyspace.NAME, COLUMNS); return createTableFromTableRowAndColumnRows(tableRow, QueryProcessor.resultify(query, serializedColumns)); } - private static CFMetaData createTableFromTablePartition(Row row) + private static CFMetaData createTableFromTablePartition(RowIterator partition) { String query = String.format("SELECT * FROM %s.%s", SystemKeyspace.NAME, COLUMNFAMILIES); - return createTableFromTableRow(QueryProcessor.resultify(query, row).one()); + return createTableFromTableRow(QueryProcessor.resultify(query, partition).one()); } /** @@ -1033,12 +983,11 @@ public class LegacySchemaTables String ksName = result.getString("keyspace_name"); String cfName = result.getString("columnfamily_name"); - Row serializedColumns = readSchemaPartitionForTable(COLUMNS, ksName, cfName); - CFMetaData cfm = createTableFromTableRowAndColumnsPartition(result, serializedColumns); + CFMetaData cfm = readSchemaPartitionForTableAndApply(COLUMNS, ksName, cfName, partition -> createTableFromTableRowAndColumnsPartition(result, partition)); - Row serializedTriggers = readSchemaPartitionForTable(TRIGGERS, ksName, cfName); - for (TriggerDefinition trigger : createTriggersFromTriggersPartition(serializedTriggers)) - cfm.addTriggerDefinition(trigger); + readSchemaPartitionForTableAndApply(TRIGGERS, ksName, cfName, + partition -> { createTriggersFromTriggersPartition(partition).forEach(trigger -> cfm.addTriggerDefinition(trigger)); return null; } + ); return cfm; } @@ -1051,35 +1000,47 @@ public class LegacySchemaTables AbstractType rawComparator = TypeParser.parse(result.getString("comparator")); AbstractType subComparator = result.has("subcomparator") ? TypeParser.parse(result.getString("subcomparator")) : null; - ColumnFamilyType cfType = ColumnFamilyType.valueOf(result.getString("type")); - AbstractType fullRawComparator = CFMetaData.makeRawAbstractType(rawComparator, subComparator); + boolean isSuper = result.getString("type").toLowerCase().equals("super"); + boolean isDense = result.getBoolean("is_dense"); + boolean isCompound = rawComparator instanceof CompositeType; - List columnDefs = createColumnsFromColumnRows(serializedColumnDefinitions, - ksName, - cfName, - fullRawComparator, - cfType == ColumnFamilyType.Super); - - boolean isDense = result.has("is_dense") - ? result.getBoolean("is_dense") - : CFMetaData.calculateIsDense(fullRawComparator, columnDefs); - - CellNameType comparator = CellNames.fromAbstractType(fullRawComparator, isDense); + // We don't really use the default validator but as we have it for backward compatibility, we use it to know if it's a counter table + AbstractType defaultValidator = TypeParser.parse(result.getString("default_validator")); + boolean isCounter = defaultValidator instanceof CounterColumnType; // if we are upgrading, we use id generated from names initially UUID cfId = result.has("cf_id") ? result.getUUID("cf_id") : CFMetaData.generateLegacyCfId(ksName, cfName); - CFMetaData cfm = new CFMetaData(ksName, cfName, cfType, comparator, cfId); - cfm.isDense(isDense); + boolean isCQLTable = !isSuper && !isDense && isCompound; + boolean isStaticCompactTable = !isDense && !isCompound; + + // Internally, compact tables have a specific layout, see CompactTables. But when upgrading from + // previous versions, they may not have the expected schema, so detect if we need to upgrade and do + // it in createColumnsFromColumnRows. + // We can remove this once we don't support upgrade from versions < 3.0. + boolean needsUpgrade = isCQLTable ? false : checkNeedsUpgrade(serializedColumnDefinitions, isSuper, isStaticCompactTable); + + List columnDefs = createColumnsFromColumnRows(serializedColumnDefinitions, + ksName, + cfName, + rawComparator, + subComparator, + isSuper, + isCQLTable, + isStaticCompactTable, + needsUpgrade); + + if (needsUpgrade) + addDefinitionForUpgrade(columnDefs, ksName, cfName, isStaticCompactTable, isSuper, rawComparator, subComparator, defaultValidator); + + CFMetaData cfm = CFMetaData.create(ksName, cfName, cfId, isDense, isCompound, isSuper, isCounter, columnDefs); cfm.readRepairChance(result.getDouble("read_repair_chance")); cfm.dcLocalReadRepairChance(result.getDouble("local_read_repair_chance")); cfm.gcGraceSeconds(result.getInt("gc_grace_seconds")); - cfm.defaultValidator(TypeParser.parse(result.getString("default_validator"))); - cfm.keyValidator(TypeParser.parse(result.getString("key_validator"))); cfm.minCompactionThreshold(result.getInt("min_compaction_threshold")); cfm.maxCompactionThreshold(result.getInt("max_compaction_threshold")); if (result.has("comment")) @@ -1107,20 +1068,86 @@ public class LegacySchemaTables cfm.bloomFilterFpChance(cfm.getBloomFilterFpChance()); if (result.has("dropped_columns")) - cfm.droppedColumns(convertDroppedColumns(result.getMap("dropped_columns", UTF8Type.instance, LongType.instance))); + { + Map types = result.has("dropped_columns_types") + ? result.getMap("dropped_columns_types", UTF8Type.instance, UTF8Type.instance) + : Collections.emptyMap(); + addDroppedColumns(cfm, result.getMap("dropped_columns", UTF8Type.instance, LongType.instance), types); + } - for (ColumnDefinition cd : columnDefs) - cfm.addOrReplaceColumnDefinition(cd); - - return cfm.rebuild(); + return cfm; } - private static Map convertDroppedColumns(Map raw) + // Should only be called on compact tables + private static boolean checkNeedsUpgrade(UntypedResultSet defs, boolean isSuper, boolean isStaticCompactTable) { - Map converted = Maps.newHashMap(); - for (Map.Entry entry : raw.entrySet()) - converted.put(new ColumnIdentifier(entry.getKey(), true), entry.getValue()); - return converted; + if (isSuper) + { + // Check if we've added the "supercolumn map" column yet or not + for (UntypedResultSet.Row row : defs) + { + if (row.getString("column_name").isEmpty()) + return false; + } + return true; + } + + // For static compact tables, we need to upgrade if the regular definitions haven't been converted to static yet, + // i.e. if we don't have a static definition yet. + if (isStaticCompactTable) + return !hasKind(defs, ColumnDefinition.Kind.STATIC); + + // For dense compact tables, we need to upgrade if we don't have a compact value definition + return !hasKind(defs, ColumnDefinition.Kind.REGULAR); + } + + private static void addDefinitionForUpgrade(List defs, + String ksName, + String cfName, + boolean isStaticCompactTable, + boolean isSuper, + AbstractType rawComparator, + AbstractType subComparator, + AbstractType defaultValidator) + { + CompactTables.DefaultNames names = CompactTables.defaultNameGenerator(defs); + + if (isSuper) + { + defs.add(ColumnDefinition.regularDef(ksName, cfName, CompactTables.SUPER_COLUMN_MAP_COLUMN_STR, MapType.getInstance(subComparator, defaultValidator, true), null)); + } + else if (isStaticCompactTable) + { + defs.add(ColumnDefinition.clusteringKeyDef(ksName, cfName, names.defaultClusteringName(), rawComparator, null)); + defs.add(ColumnDefinition.regularDef(ksName, cfName, names.defaultCompactValueName(), defaultValidator, null)); + } + else + { + // For dense compact tables, we get here if we don't have a compact value column, in which case we should add it + // (we use EmptyType to recognize that the compact value was not declared by the use (see CreateTableStatement too)) + defs.add(ColumnDefinition.regularDef(ksName, cfName, names.defaultCompactValueName(), EmptyType.instance, null)); + } + } + + private static boolean hasKind(UntypedResultSet defs, ColumnDefinition.Kind kind) + { + for (UntypedResultSet.Row row : defs) + { + if (deserializeKind(row.getString("type")) == kind) + return true; + } + return false; + } + + private static void addDroppedColumns(CFMetaData cfm, Map droppedTimes, Map types) + { + for (Map.Entry entry : droppedTimes.entrySet()) + { + String name = entry.getKey(); + long time = entry.getValue(); + AbstractType type = types.containsKey(name) ? TypeParser.parse(types.get(name)) : null; + cfm.getDroppedColumns().put(ColumnIdentifier.getInterned(name, true), new CFMetaData.DroppedColumn(type, time)); + } } /* @@ -1129,50 +1156,59 @@ public class LegacySchemaTables private static void addColumnToSchemaMutation(CFMetaData table, ColumnDefinition column, long timestamp, Mutation mutation) { - ColumnFamily cells = mutation.addOrGet(Columns); - Composite prefix = Columns.comparator.make(table.cfName, column.name.toString()); - CFRowAdder adder = new CFRowAdder(cells, prefix, timestamp); + RowUpdateBuilder adder = new RowUpdateBuilder(Columns, timestamp, mutation) + .clustering(table.cfName, column.name.toString()); adder.add("validator", column.type.toString()); - adder.add("type", serializeKind(column.kind)); + adder.add("type", serializeKind(column.kind, table.isDense())); adder.add("component_index", column.isOnAllComponents() ? null : column.position()); adder.add("index_name", column.getIndexName()); adder.add("index_type", column.getIndexType() == null ? null : column.getIndexType().toString()); adder.add("index_options", json(column.getIndexOptions())); + + adder.build(); } - private static String serializeKind(ColumnDefinition.Kind kind) + private static String serializeKind(ColumnDefinition.Kind kind, boolean isDense) { - // For backward compatibility we need to special case CLUSTERING_COLUMN - return kind == ColumnDefinition.Kind.CLUSTERING_COLUMN ? "clustering_key" : kind.toString().toLowerCase(); + // For backward compatibility, we special case CLUSTERING_COLUMN and the case where the table is dense. + if (kind == ColumnDefinition.Kind.CLUSTERING_COLUMN) + return "clustering_key"; + + if (kind == ColumnDefinition.Kind.REGULAR && isDense) + return "compact_value"; + + return kind.toString().toLowerCase(); } - private static ColumnDefinition.Kind deserializeKind(String kind) + public static ColumnDefinition.Kind deserializeKind(String kind) { if (kind.equalsIgnoreCase("clustering_key")) return ColumnDefinition.Kind.CLUSTERING_COLUMN; + if (kind.equalsIgnoreCase("compact_value")) + return ColumnDefinition.Kind.REGULAR; return Enum.valueOf(ColumnDefinition.Kind.class, kind.toUpperCase()); } private static void dropColumnFromSchemaMutation(CFMetaData table, ColumnDefinition column, long timestamp, Mutation mutation) { - ColumnFamily cells = mutation.addOrGet(Columns); - int ldt = (int) (System.currentTimeMillis() / 1000); - // Note: we do want to use name.toString(), not name.bytes directly for backward compatibility (For CQL3, this won't make a difference). - Composite prefix = Columns.comparator.make(table.cfName, column.name.toString()); - cells.addAtom(new RangeTombstone(prefix, prefix.end(), timestamp, ldt)); + RowUpdateBuilder.deleteRow(Columns, timestamp, mutation, table.cfName, column.name.toString()); } private static List createColumnsFromColumnRows(UntypedResultSet rows, String keyspace, String table, AbstractType rawComparator, - boolean isSuper) + AbstractType rawSubComparator, + boolean isSuper, + boolean isCQLTable, + boolean isStaticCompactTable, + boolean needsUpgrade) { List columns = new ArrayList<>(); for (UntypedResultSet.Row row : rows) - columns.add(createColumnFromColumnRow(row, keyspace, table, rawComparator, isSuper)); + columns.add(createColumnFromColumnRow(row, keyspace, table, rawComparator, rawSubComparator, isSuper, isCQLTable, isStaticCompactTable, needsUpgrade)); return columns; } @@ -1180,22 +1216,26 @@ public class LegacySchemaTables String keyspace, String table, AbstractType rawComparator, - boolean isSuper) + AbstractType rawSubComparator, + boolean isSuper, + boolean isCQLTable, + boolean isStaticCompactTable, + boolean needsUpgrade) { ColumnDefinition.Kind kind = deserializeKind(row.getString("type")); + if (needsUpgrade && isStaticCompactTable && kind == ColumnDefinition.Kind.REGULAR) + kind = ColumnDefinition.Kind.STATIC; Integer componentIndex = null; if (row.has("component_index")) componentIndex = row.getInt("component_index"); - else if (kind == ColumnDefinition.Kind.CLUSTERING_COLUMN && isSuper) - componentIndex = 1; // A ColumnDefinition for super columns applies to the column component // Note: we save the column name as string, but we should not assume that it is an UTF8 name, we // we need to use the comparator fromString method - AbstractType comparator = kind == ColumnDefinition.Kind.REGULAR - ? getComponentComparator(rawComparator, componentIndex) - : UTF8Type.instance; - ColumnIdentifier name = new ColumnIdentifier(comparator.fromString(row.getString("column_name")), comparator); + AbstractType comparator = isCQLTable + ? UTF8Type.instance + : CompactTables.columnDefinitionComparator(kind, isSuper, rawComparator, rawSubComparator); + ColumnIdentifier name = ColumnIdentifier.getInterned(comparator.fromString(row.getString("column_name")), comparator); AbstractType validator = parseType(row.getString("validator")); @@ -1214,32 +1254,21 @@ public class LegacySchemaTables return new ColumnDefinition(keyspace, table, name, validator, indexType, indexOptions, indexName, componentIndex, kind); } - private static AbstractType getComponentComparator(AbstractType rawComparator, Integer componentIndex) - { - return (componentIndex == null || (componentIndex == 0 && !(rawComparator instanceof CompositeType))) - ? rawComparator - : ((CompositeType)rawComparator).types.get(componentIndex); - } - /* * Trigger metadata serialization/deserialization. */ private static void addTriggerToSchemaMutation(CFMetaData table, TriggerDefinition trigger, long timestamp, Mutation mutation) { - ColumnFamily cells = mutation.addOrGet(Triggers); - Composite prefix = Triggers.comparator.make(table.cfName, trigger.name); - CFRowAdder adder = new CFRowAdder(cells, prefix, timestamp); - adder.addMapEntry("trigger_options", "class", trigger.classOption); + new RowUpdateBuilder(Triggers, timestamp, mutation) + .clustering(table.cfName, trigger.name) + .addMapEntry("trigger_options", "class", trigger.classOption) + .build(); } private static void dropTriggerFromSchemaMutation(CFMetaData table, TriggerDefinition trigger, long timestamp, Mutation mutation) { - ColumnFamily cells = mutation.addOrGet(Triggers); - int ldt = (int) (System.currentTimeMillis() / 1000); - - Composite prefix = Triggers.comparator.make(table.cfName, trigger.name); - cells.addAtom(new RangeTombstone(prefix, prefix.end(), timestamp, ldt)); + RowUpdateBuilder.deleteRow(Triggers, timestamp, mutation, table.cfName, trigger.name); } /** @@ -1248,7 +1277,7 @@ public class LegacySchemaTables * @param partition storage-level partition containing the trigger definitions * @return the list of processed TriggerDefinitions */ - private static List createTriggersFromTriggersPartition(Row partition) + private static List createTriggersFromTriggersPartition(RowIterator partition) { List triggers = new ArrayList<>(); String query = String.format("SELECT * FROM %s.%s", SystemKeyspace.NAME, TRIGGERS); @@ -1275,48 +1304,37 @@ public class LegacySchemaTables private static void addFunctionToSchemaMutation(UDFunction function, long timestamp, Mutation mutation) { - ColumnFamily cells = mutation.addOrGet(Functions); - Composite prefix = Functions.comparator.make(function.name().name, functionSignatureWithTypes(function)); - CFRowAdder adder = new CFRowAdder(cells, prefix, timestamp); - - adder.resetCollection("argument_names"); - adder.resetCollection("argument_types"); - - for (int i = 0; i < function.argNames().size(); i++) - { - adder.addListEntry("argument_names", function.argNames().get(i).bytes); - adder.addListEntry("argument_types", function.argTypes().get(i).toString()); - } + RowUpdateBuilder adder = new RowUpdateBuilder(Functions, timestamp, mutation) + .clustering(function.name().name, functionSignatureWithTypes(function)); adder.add("body", function.body()); adder.add("language", function.language()); adder.add("return_type", function.returnType().toString()); adder.add("called_on_null_input", function.isCalledOnNullInput()); + + adder.resetCollection("argument_names"); + adder.resetCollection("argument_types"); + for (int i = 0; i < function.argNames().size(); i++) + { + adder.addListEntry("argument_names", function.argNames().get(i).bytes); + adder.addListEntry("argument_types", function.argTypes().get(i).toString()); + } + adder.build(); } public static Mutation makeDropFunctionMutation(KSMetaData keyspace, UDFunction function, long timestamp) { // Include the serialized keyspace in case the target node missed a CREATE KEYSPACE migration (see CASSANDRA-5631). Mutation mutation = makeCreateKeyspaceMutation(keyspace, timestamp, false); - - ColumnFamily cells = mutation.addOrGet(Functions); - int ldt = (int) (System.currentTimeMillis() / 1000); - - Composite prefix = Functions.comparator.make(function.name().name, functionSignatureWithTypes(function)); - cells.addAtom(new RangeTombstone(prefix, prefix.end(), timestamp, ldt)); - - return mutation; + return RowUpdateBuilder.deleteRow(Functions, timestamp, mutation, function.name().name, functionSignatureWithTypes(function)); } - private static Map createFunctionsFromFunctionsPartition(Row partition) + private static Collection createFunctionsFromFunctionsPartition(RowIterator partition) { - Map functions = new HashMap<>(); + List functions = new ArrayList<>(); String query = String.format("SELECT * FROM %s.%s", SystemKeyspace.NAME, FUNCTIONS); for (UntypedResultSet.Row row : QueryProcessor.resultify(query, partition)) - { - UDFunction function = createFunctionFromFunctionRow(row); - functions.put(functionSignatureWithNameAndTypes(function), function); - } + functions.add(createFunctionFromFunctionRow(row)); return functions; } @@ -1387,9 +1405,8 @@ public class LegacySchemaTables private static void addAggregateToSchemaMutation(UDAggregate aggregate, long timestamp, Mutation mutation) { - ColumnFamily cells = mutation.addOrGet(Aggregates); - Composite prefix = Aggregates.comparator.make(aggregate.name().name, functionSignatureWithTypes(aggregate)); - CFRowAdder adder = new CFRowAdder(cells, prefix, timestamp); + RowUpdateBuilder adder = new RowUpdateBuilder(Aggregates, timestamp, mutation) + .clustering(aggregate.name().name, functionSignatureWithTypes(aggregate)); adder.resetCollection("argument_types"); adder.add("return_type", aggregate.returnType().toString()); @@ -1403,17 +1420,16 @@ public class LegacySchemaTables for (AbstractType argType : aggregate.argTypes()) adder.addListEntry("argument_types", argType.toString()); + + adder.build(); } - private static Map createAggregatesFromAggregatesPartition(Row partition) + private static Collection createAggregatesFromAggregatesPartition(RowIterator partition) { - Map aggregates = new HashMap<>(); + List aggregates = new ArrayList<>(); String query = String.format("SELECT * FROM %s.%s", SystemKeyspace.NAME, AGGREGATES); for (UntypedResultSet.Row row : QueryProcessor.resultify(query, partition)) - { - UDAggregate aggregate = createAggregateFromAggregateRow(row); - aggregates.put(functionSignatureWithNameAndTypes(aggregate), aggregate); - } + aggregates.add(createAggregateFromAggregateRow(row)); return aggregates; } @@ -1475,14 +1491,7 @@ public class LegacySchemaTables { // Include the serialized keyspace in case the target node missed a CREATE KEYSPACE migration (see CASSANDRA-5631). Mutation mutation = makeCreateKeyspaceMutation(keyspace, timestamp, false); - - ColumnFamily cells = mutation.addOrGet(Aggregates); - int ldt = (int) (System.currentTimeMillis() / 1000); - - Composite prefix = Aggregates.comparator.make(aggregate.name().name, functionSignatureWithTypes(aggregate)); - cells.addAtom(new RangeTombstone(prefix, prefix.end(), timestamp, ldt)); - - return mutation; + return RowUpdateBuilder.deleteRow(Aggregates, timestamp, mutation, aggregate.name().name, functionSignatureWithTypes(aggregate)); } private static AbstractType parseType(String str) @@ -1515,5 +1524,4 @@ public class LegacySchemaTables strList.add(argType.asCQL3Type().toString()); return list.decompose(strList); } - } diff --git a/src/java/org/apache/cassandra/serializers/ListSerializer.java b/src/java/org/apache/cassandra/serializers/ListSerializer.java index aeee2b9dd7..b1c5508156 100644 --- a/src/java/org/apache/cassandra/serializers/ListSerializer.java +++ b/src/java/org/apache/cassandra/serializers/ListSerializer.java @@ -143,14 +143,16 @@ public class ListSerializer extends CollectionSerializer> { StringBuilder sb = new StringBuilder(); boolean isFirst = true; + sb.append('['); for (T element : value) { if (isFirst) isFirst = false; else - sb.append("; "); + sb.append(", "); sb.append(elements.toString(element)); } + sb.append(']'); return sb.toString(); } diff --git a/src/java/org/apache/cassandra/serializers/MapSerializer.java b/src/java/org/apache/cassandra/serializers/MapSerializer.java index 8350f66779..7d81598078 100644 --- a/src/java/org/apache/cassandra/serializers/MapSerializer.java +++ b/src/java/org/apache/cassandra/serializers/MapSerializer.java @@ -84,7 +84,7 @@ public class MapSerializer extends CollectionSerializer> } catch (BufferUnderflowException e) { - throw new MarshalException("Not enough bytes to read a set"); + throw new MarshalException("Not enough bytes to read a map"); } } @@ -150,19 +150,19 @@ public class MapSerializer extends CollectionSerializer> public String toString(Map value) { StringBuilder sb = new StringBuilder(); + sb.append('{'); boolean isFirst = true; for (Map.Entry element : value.entrySet()) { if (isFirst) isFirst = false; else - sb.append("; "); - sb.append('('); + sb.append(", "); sb.append(keys.toString(element.getKey())); - sb.append(", "); + sb.append(": "); sb.append(values.toString(element.getValue())); - sb.append(')'); } + sb.append('}'); return sb.toString(); } diff --git a/src/java/org/apache/cassandra/serializers/SetSerializer.java b/src/java/org/apache/cassandra/serializers/SetSerializer.java index 21f507558d..710863006f 100644 --- a/src/java/org/apache/cassandra/serializers/SetSerializer.java +++ b/src/java/org/apache/cassandra/serializers/SetSerializer.java @@ -94,13 +94,14 @@ public class SetSerializer extends CollectionSerializer> } catch (BufferUnderflowException e) { - throw new MarshalException("Not enough bytes to read a list"); + throw new MarshalException("Not enough bytes to read a set"); } } public String toString(Set value) { StringBuilder sb = new StringBuilder(); + sb.append('{'); boolean isFirst = true; for (T element : value) { @@ -110,10 +111,11 @@ public class SetSerializer extends CollectionSerializer> } else { - sb.append("; "); + sb.append(", "); } sb.append(elements.toString(element)); } + sb.append('}'); return sb.toString(); } diff --git a/src/java/org/apache/cassandra/service/AbstractReadExecutor.java b/src/java/org/apache/cassandra/service/AbstractReadExecutor.java index 3aab12f534..6696e10a5f 100644 --- a/src/java/org/apache/cassandra/service/AbstractReadExecutor.java +++ b/src/java/org/apache/cassandra/service/AbstractReadExecutor.java @@ -29,14 +29,13 @@ import org.slf4j.LoggerFactory; import org.apache.cassandra.concurrent.Stage; import org.apache.cassandra.concurrent.StageManager; import org.apache.cassandra.config.CFMetaData.SpeculativeRetry.RetryType; -import org.apache.cassandra.config.Schema; import org.apache.cassandra.config.ReadRepairDecision; import org.apache.cassandra.db.ColumnFamilyStore; import org.apache.cassandra.db.ConsistencyLevel; import org.apache.cassandra.db.ReadCommand; -import org.apache.cassandra.db.ReadResponse; -import org.apache.cassandra.db.Row; +import org.apache.cassandra.db.SinglePartitionReadCommand; import org.apache.cassandra.db.Keyspace; +import org.apache.cassandra.db.partitions.PartitionIterator; import org.apache.cassandra.exceptions.ReadFailureException; import org.apache.cassandra.exceptions.ReadTimeoutException; import org.apache.cassandra.exceptions.UnavailableException; @@ -62,22 +61,15 @@ public abstract class AbstractReadExecutor protected final ReadCommand command; protected final List targetReplicas; - protected final RowDigestResolver resolver; - protected final ReadCallback handler; + protected final ReadCallback handler; protected final TraceState traceState; - AbstractReadExecutor(ReadCommand command, ConsistencyLevel consistencyLevel, List targetReplicas) + AbstractReadExecutor(Keyspace keyspace, ReadCommand command, ConsistencyLevel consistencyLevel, List targetReplicas) { this.command = command; this.targetReplicas = targetReplicas; - resolver = new RowDigestResolver(command.ksName, command.key, targetReplicas.size()); - traceState = Tracing.instance.get(); - handler = new ReadCallback<>(resolver, consistencyLevel, command, targetReplicas); - } - - private static boolean isLocalRequest(InetAddress replica) - { - return replica.equals(FBUtilities.getBroadcastAddress()) && StorageProxy.OPTIMIZE_LOCAL_REQUESTS; + this.handler = new ReadCallback(new DigestResolver(keyspace, command, consistencyLevel, targetReplicas.size()), consistencyLevel, command, targetReplicas); + this.traceState = Tracing.instance.get(); } protected void makeDataRequests(Iterable endpoints) @@ -98,7 +90,7 @@ public abstract class AbstractReadExecutor for (InetAddress endpoint : endpoints) { - if (isLocalRequest(endpoint)) + if (StorageProxy.canDoLocalRequest(endpoint)) { hasLocalEndpoint = true; continue; @@ -142,7 +134,7 @@ public abstract class AbstractReadExecutor * wait for an answer. Blocks until success or timeout, so it is caller's * responsibility to call maybeTryAdditionalReplicas first. */ - public Row get() throws ReadFailureException, ReadTimeoutException, DigestMismatchException + public PartitionIterator get() throws ReadFailureException, ReadTimeoutException, DigestMismatchException { return handler.get(); } @@ -150,11 +142,11 @@ public abstract class AbstractReadExecutor /** * @return an executor appropriate for the configured speculative read policy */ - public static AbstractReadExecutor getReadExecutor(ReadCommand command, ConsistencyLevel consistencyLevel) throws UnavailableException + public static AbstractReadExecutor getReadExecutor(SinglePartitionReadCommand command, ConsistencyLevel consistencyLevel) throws UnavailableException { - Keyspace keyspace = Keyspace.open(command.ksName); - List allReplicas = StorageProxy.getLiveSortedEndpoints(keyspace, command.key); - ReadRepairDecision repairDecision = Schema.instance.getCFMetaData(command.ksName, command.cfName).newReadRepairDecision(); + Keyspace keyspace = Keyspace.open(command.metadata().ksName); + List allReplicas = StorageProxy.getLiveSortedEndpoints(keyspace, command.partitionKey()); + ReadRepairDecision repairDecision = command.metadata().newReadRepairDecision(); List targetReplicas = consistencyLevel.filterForQuery(keyspace, allReplicas, repairDecision); // Throw UAE early if we don't have enough replicas. @@ -166,19 +158,19 @@ public abstract class AbstractReadExecutor ReadRepairMetrics.attempted.mark(); } - ColumnFamilyStore cfs = keyspace.getColumnFamilyStore(command.cfName); + ColumnFamilyStore cfs = keyspace.getColumnFamilyStore(command.metadata().cfId); RetryType retryType = cfs.metadata.getSpeculativeRetry().type; // Speculative retry is disabled *OR* there are simply no extra replicas to speculate. if (retryType == RetryType.NONE || consistencyLevel.blockFor(keyspace) == allReplicas.size()) - return new NeverSpeculatingReadExecutor(command, consistencyLevel, targetReplicas); + return new NeverSpeculatingReadExecutor(keyspace, command, consistencyLevel, targetReplicas); if (targetReplicas.size() == allReplicas.size()) { // CL.ALL, RRD.GLOBAL or RRD.DC_LOCAL and a single-DC. // We are going to contact every node anyway, so ask for 2 full data requests instead of 1, for redundancy // (same amount of requests in total, but we turn 1 digest request into a full blown data request). - return new AlwaysSpeculatingReadExecutor(cfs, command, consistencyLevel, targetReplicas); + return new AlwaysSpeculatingReadExecutor(keyspace, cfs, command, consistencyLevel, targetReplicas); } // RRD.NONE or RRD.DC_LOCAL w/ multiple DCs. @@ -199,16 +191,16 @@ public abstract class AbstractReadExecutor targetReplicas.add(extraReplica); if (retryType == RetryType.ALWAYS) - return new AlwaysSpeculatingReadExecutor(cfs, command, consistencyLevel, targetReplicas); + return new AlwaysSpeculatingReadExecutor(keyspace, cfs, command, consistencyLevel, targetReplicas); else // PERCENTILE or CUSTOM. - return new SpeculatingReadExecutor(cfs, command, consistencyLevel, targetReplicas); + return new SpeculatingReadExecutor(keyspace, cfs, command, consistencyLevel, targetReplicas); } private static class NeverSpeculatingReadExecutor extends AbstractReadExecutor { - public NeverSpeculatingReadExecutor(ReadCommand command, ConsistencyLevel consistencyLevel, List targetReplicas) + public NeverSpeculatingReadExecutor(Keyspace keyspace, ReadCommand command, ConsistencyLevel consistencyLevel, List targetReplicas) { - super(command, consistencyLevel, targetReplicas); + super(keyspace, command, consistencyLevel, targetReplicas); } public void executeAsync() @@ -234,12 +226,13 @@ public abstract class AbstractReadExecutor private final ColumnFamilyStore cfs; private volatile boolean speculated = false; - public SpeculatingReadExecutor(ColumnFamilyStore cfs, + public SpeculatingReadExecutor(Keyspace keyspace, + ColumnFamilyStore cfs, ReadCommand command, ConsistencyLevel consistencyLevel, List targetReplicas) { - super(command, consistencyLevel, targetReplicas); + super(keyspace, command, consistencyLevel, targetReplicas); this.cfs = cfs; } @@ -278,7 +271,7 @@ public abstract class AbstractReadExecutor { // Could be waiting on the data, or on enough digests. ReadCommand retryCommand = command; - if (resolver.getData() != null) + if (handler.resolver.isDataPresent()) retryCommand = command.copy().setIsDigestQuery(true); InetAddress extraReplica = Iterables.getLast(targetReplicas); @@ -304,12 +297,13 @@ public abstract class AbstractReadExecutor { private final ColumnFamilyStore cfs; - public AlwaysSpeculatingReadExecutor(ColumnFamilyStore cfs, + public AlwaysSpeculatingReadExecutor(Keyspace keyspace, + ColumnFamilyStore cfs, ReadCommand command, ConsistencyLevel consistencyLevel, List targetReplicas) { - super(command, consistencyLevel, targetReplicas); + super(keyspace, command, consistencyLevel, targetReplicas); this.cfs = cfs; } diff --git a/src/java/org/apache/cassandra/service/AsyncRepairCallback.java b/src/java/org/apache/cassandra/service/AsyncRepairCallback.java index 6ac765b837..dec5319056 100644 --- a/src/java/org/apache/cassandra/service/AsyncRepairCallback.java +++ b/src/java/org/apache/cassandra/service/AsyncRepairCallback.java @@ -29,11 +29,11 @@ import org.apache.cassandra.utils.WrappedRunnable; public class AsyncRepairCallback implements IAsyncCallback { - private final RowDataResolver repairResolver; + private final DataResolver repairResolver; private final int blockfor; protected final AtomicInteger received = new AtomicInteger(0); - public AsyncRepairCallback(RowDataResolver repairResolver, int blockfor) + public AsyncRepairCallback(DataResolver repairResolver, int blockfor) { this.repairResolver = repairResolver; this.blockfor = blockfor; diff --git a/src/java/org/apache/cassandra/service/CASRequest.java b/src/java/org/apache/cassandra/service/CASRequest.java index 3d86637c3c..1db100d3a7 100644 --- a/src/java/org/apache/cassandra/service/CASRequest.java +++ b/src/java/org/apache/cassandra/service/CASRequest.java @@ -17,8 +17,9 @@ */ package org.apache.cassandra.service; -import org.apache.cassandra.db.ColumnFamily; -import org.apache.cassandra.db.filter.IDiskAtomFilter; +import org.apache.cassandra.db.SinglePartitionReadCommand; +import org.apache.cassandra.db.partitions.FilteredPartition; +import org.apache.cassandra.db.partitions.PartitionUpdate; import org.apache.cassandra.exceptions.InvalidRequestException; /** @@ -27,19 +28,19 @@ import org.apache.cassandra.exceptions.InvalidRequestException; public interface CASRequest { /** - * The filter to use to fetch the value to compare for the CAS. + * The command to use to fetch the value to compare for the CAS. */ - public IDiskAtomFilter readFilter(); + public SinglePartitionReadCommand readCommand(int nowInSec); /** * Returns whether the provided CF, that represents the values fetched using the * readFilter(), match the CAS conditions this object stands for. */ - public boolean appliesTo(ColumnFamily current) throws InvalidRequestException; + public boolean appliesTo(FilteredPartition current) throws InvalidRequestException; /** * The updates to perform of a CAS success. The values fetched using the readFilter() * are passed as argument. */ - public ColumnFamily makeUpdates(ColumnFamily current) throws InvalidRequestException; + public PartitionUpdate makeUpdates(FilteredPartition current) throws InvalidRequestException; } diff --git a/src/java/org/apache/cassandra/service/CacheService.java b/src/java/org/apache/cassandra/service/CacheService.java index a775627699..e82e8a420d 100644 --- a/src/java/org/apache/cassandra/service/CacheService.java +++ b/src/java/org/apache/cassandra/service/CacheService.java @@ -43,17 +43,20 @@ import org.apache.cassandra.cache.AutoSavingCache.CacheSerializer; import org.apache.cassandra.concurrent.Stage; import org.apache.cassandra.concurrent.StageManager; import org.apache.cassandra.config.CFMetaData; +import org.apache.cassandra.config.ColumnDefinition; import org.apache.cassandra.config.DatabaseDescriptor; import org.apache.cassandra.config.Schema; import org.apache.cassandra.db.*; -import org.apache.cassandra.db.composites.CellName; +import org.apache.cassandra.db.rows.*; +import org.apache.cassandra.db.filter.*; +import org.apache.cassandra.db.partitions.ArrayBackedCachedPartition; +import org.apache.cassandra.db.partitions.CachedPartition; import org.apache.cassandra.db.context.CounterContext; -import org.apache.cassandra.db.filter.QueryFilter; -import org.apache.cassandra.io.sstable.Descriptor; import org.apache.cassandra.io.util.DataOutputPlus; import org.apache.cassandra.utils.ByteBufferUtil; import org.apache.cassandra.utils.FBUtilities; import org.apache.cassandra.utils.Pair; +import org.apache.cassandra.utils.concurrent.OpOrder; public class CacheService implements CacheServiceMBean { @@ -362,24 +365,45 @@ public class CacheService implements CacheServiceMBean public Future> deserialize(DataInputStream in, final ColumnFamilyStore cfs) throws IOException { final ByteBuffer partitionKey = ByteBufferUtil.readWithLength(in); - final CellName cellName = cfs.metadata.comparator.cellFromByteBuffer(ByteBufferUtil.readWithLength(in)); + final ByteBuffer cellName = ByteBufferUtil.readWithLength(in); return StageManager.getStage(Stage.READ).submit(new Callable>() { public Pair call() throws Exception { DecoratedKey key = cfs.partitioner.decorateKey(partitionKey); - QueryFilter filter = QueryFilter.getNamesFilter(key, - cfs.metadata.cfName, - FBUtilities.singleton(cellName, cfs.metadata.comparator), - Long.MIN_VALUE); - ColumnFamily cf = cfs.getTopLevelColumns(filter, Integer.MIN_VALUE); - if (cf == null) - return null; - Cell cell = cf.getColumn(cellName); - if (cell == null || !cell.isLive(Long.MIN_VALUE)) - return null; - ClockAndCount clockAndCount = CounterContext.instance().getLocalClockAndCount(cell.value()); - return Pair.create(CounterCacheKey.create(cfs.metadata.cfId, partitionKey, cellName), clockAndCount); + LegacyLayout.LegacyCellName name = LegacyLayout.decodeCellName(cfs.metadata, cellName); + ColumnDefinition column = name.column; + CellPath path = name.collectionElement == null ? null : CellPath.create(name.collectionElement); + + int nowInSec = FBUtilities.nowInSeconds(); + ColumnFilter.Builder builder = ColumnFilter.selectionBuilder(); + if (path == null) + builder.add(column); + else + builder.select(column, path); + + ClusteringIndexFilter filter = new ClusteringIndexNamesFilter(FBUtilities.singleton(name.clustering, cfs.metadata.comparator), false); + SinglePartitionReadCommand cmd = SinglePartitionReadCommand.create(cfs.metadata, nowInSec, key, builder.build(), filter); + try (OpOrder.Group op = cfs.readOrdering.start(); RowIterator iter = UnfilteredRowIterators.filter(cmd.queryMemtableAndDisk(cfs, op), nowInSec)) + { + Cell cell; + if (column.isStatic()) + { + cell = iter.staticRow().getCell(column); + } + else + { + if (!iter.hasNext()) + return null; + cell = iter.next().getCell(column); + } + + if (cell == null) + return null; + + ClockAndCount clockAndCount = CounterContext.instance().getLocalClockAndCount(cell.value()); + return Pair.create(CounterCacheKey.create(cfs.metadata.cfId, partitionKey, name.clustering, column, path), clockAndCount); + } } }); } @@ -395,14 +419,19 @@ public class CacheService implements CacheServiceMBean public Future> deserialize(DataInputStream in, final ColumnFamilyStore cfs) throws IOException { final ByteBuffer buffer = ByteBufferUtil.readWithLength(in); + final int rowsToCache = cfs.metadata.getCaching().rowCache.rowsToCache; + return StageManager.getStage(Stage.READ).submit(new Callable>() { public Pair call() throws Exception { DecoratedKey key = cfs.partitioner.decorateKey(buffer); - QueryFilter cacheFilter = new QueryFilter(key, cfs.getColumnFamilyName(), cfs.readFilterForCache(), Integer.MIN_VALUE); - ColumnFamily data = cfs.getTopLevelColumns(cacheFilter, Integer.MIN_VALUE); - return Pair.create(new RowCacheKey(cfs.metadata.cfId, key), (IRowCacheEntry) data); + int nowInSec = FBUtilities.nowInSeconds(); + try (OpOrder.Group op = cfs.readOrdering.start(); UnfilteredRowIterator iter = SinglePartitionReadCommand.fullPartitionRead(cfs.metadata, nowInSec, key).queryMemtableAndDisk(cfs, op)) + { + CachedPartition toCache = ArrayBackedCachedPartition.create(DataLimits.cqlLimits(rowsToCache).filter(iter, nowInSec), nowInSec); + return Pair.create(new RowCacheKey(cfs.metadata.cfId, key), (IRowCacheEntry)toCache); + } } }); } @@ -423,7 +452,7 @@ public class CacheService implements CacheServiceMBean ByteBufferUtil.writeWithLength(key.key, out); out.writeInt(key.desc.generation); out.writeBoolean(true); - key.desc.getFormat().getIndexSerializer(cfm).serialize(entry, out); + key.desc.getFormat().getIndexSerializer(cfm, key.desc.version, SerializationHeader.forKeyCache(cfm)).serialize(entry, out); } public Future> deserialize(DataInputStream input, ColumnFamilyStore cfs) throws IOException @@ -443,7 +472,10 @@ public class CacheService implements CacheServiceMBean RowIndexEntry.Serializer.skipPromotedIndex(input); return null; } - RowIndexEntry entry = reader.descriptor.getFormat().getIndexSerializer(reader.metadata).deserialize(input, reader.descriptor.version); + RowIndexEntry.IndexSerializer indexSerializer = reader.descriptor.getFormat().getIndexSerializer(reader.metadata, + reader.descriptor.version, + SerializationHeader.forKeyCache(cfs.metadata)); + RowIndexEntry entry = indexSerializer.deserialize(input); return Futures.immediateFuture(Pair.create(new KeyCacheKey(cfs.metadata.cfId, reader.descriptor, key), entry)); } diff --git a/src/java/org/apache/cassandra/service/DataResolver.java b/src/java/org/apache/cassandra/service/DataResolver.java new file mode 100644 index 0000000000..b2d195446e --- /dev/null +++ b/src/java/org/apache/cassandra/service/DataResolver.java @@ -0,0 +1,428 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.cassandra.service; + +import java.net.InetAddress; +import java.util.*; +import java.util.concurrent.TimeoutException; + +import org.apache.cassandra.concurrent.Stage; +import org.apache.cassandra.concurrent.StageManager; +import org.apache.cassandra.config.ColumnDefinition; +import org.apache.cassandra.config.DatabaseDescriptor; +import org.apache.cassandra.db.*; +import org.apache.cassandra.db.rows.*; +import org.apache.cassandra.db.filter.*; +import org.apache.cassandra.db.partitions.*; +import org.apache.cassandra.exceptions.ReadTimeoutException; +import org.apache.cassandra.net.AsyncOneResponse; +import org.apache.cassandra.net.MessageIn; +import org.apache.cassandra.net.MessageOut; +import org.apache.cassandra.net.MessagingService; +import org.apache.cassandra.tracing.Tracing; +import org.apache.cassandra.utils.FBUtilities; + +public class DataResolver extends ResponseResolver +{ + private final List repairResults = Collections.synchronizedList(new ArrayList()); + + public DataResolver(Keyspace keyspace, ReadCommand command, ConsistencyLevel consistency, int maxResponseCount) + { + super(keyspace, command, consistency, maxResponseCount); + } + + public PartitionIterator getData() + { + ReadResponse response = responses.iterator().next().payload; + return UnfilteredPartitionIterators.filter(response.makeIterator(), command.nowInSec()); + } + + public PartitionIterator resolve() + { + // We could get more responses while this method runs, which is ok (we're happy to ignore any response not here + // at the beginning of this method), so grab the response count once and use that through the method. + int count = responses.size(); + List iters = new ArrayList<>(count); + InetAddress[] sources = new InetAddress[count]; + for (int i = 0; i < count; i++) + { + MessageIn msg = responses.get(i); + iters.add(msg.payload.makeIterator()); + sources[i] = msg.from; + } + + // Even though every responses should honor the limit, we might have more than requested post reconciliation, + // so ensure we're respecting the limit. + DataLimits.Counter counter = command.limits().newCounter(command.nowInSec(), true); + return new CountingPartitionIterator(mergeWithShortReadProtection(iters, sources, counter), counter); + } + + private PartitionIterator mergeWithShortReadProtection(List results, InetAddress[] sources, DataLimits.Counter resultCounter) + { + // If we have only one results, there is no read repair to do and we can't get short reads + if (results.size() == 1) + return UnfilteredPartitionIterators.filter(results.get(0), command.nowInSec()); + + UnfilteredPartitionIterators.MergeListener listener = new RepairMergeListener(sources); + + // So-called "short reads" stems from nodes returning only a subset of the results they have for a partition due to the limit, + // but that subset not being enough post-reconciliation. So if we don't have limit, don't bother. + if (command.limits().isUnlimited()) + return UnfilteredPartitionIterators.mergeAndFilter(results, command.nowInSec(), listener); + + for (int i = 0; i < results.size(); i++) + results.set(i, new ShortReadProtectedIterator(sources[i], results.get(i), resultCounter)); + + return UnfilteredPartitionIterators.mergeAndFilter(results, command.nowInSec(), listener); + } + + private class RepairMergeListener implements UnfilteredPartitionIterators.MergeListener + { + private final InetAddress[] sources; + + public RepairMergeListener(InetAddress[] sources) + { + this.sources = sources; + } + + public UnfilteredRowIterators.MergeListener getRowMergeListener(DecoratedKey partitionKey, List versions) + { + return new MergeListener(partitionKey, columns(versions), isReversed(versions)); + } + + private PartitionColumns columns(List versions) + { + Columns statics = Columns.NONE; + Columns regulars = Columns.NONE; + for (UnfilteredRowIterator iter : versions) + { + if (iter == null) + continue; + + PartitionColumns cols = iter.columns(); + statics = statics.mergeTo(cols.statics); + regulars = regulars.mergeTo(cols.regulars); + } + return new PartitionColumns(statics, regulars); + } + + private boolean isReversed(List versions) + { + assert !versions.isEmpty(); + // Everything will be in the same order + return versions.get(0).isReverseOrder(); + } + + public void close() + { + try + { + FBUtilities.waitOnFutures(repairResults, DatabaseDescriptor.getWriteRpcTimeout()); + } + catch (TimeoutException ex) + { + // We got all responses, but timed out while repairing + int blockFor = consistency.blockFor(keyspace); + if (Tracing.isTracing()) + Tracing.trace("Timed out while read-repairing after receiving all {} data and digest responses", blockFor); + else + logger.debug("Timeout while read-repairing after receiving all {} data and digest responses", blockFor); + + throw new ReadTimeoutException(consistency, blockFor-1, blockFor, true); + } + } + + private class MergeListener implements UnfilteredRowIterators.MergeListener + { + private final DecoratedKey partitionKey; + private final PartitionColumns columns; + private final boolean isReversed; + private final PartitionUpdate[] repairs = new PartitionUpdate[sources.length]; + + private final Row.Writer[] currentRows = new Row.Writer[sources.length]; + private Clustering currentClustering; + private ColumnDefinition currentColumn; + + private final Slice.Bound[] markerOpen = new Slice.Bound[sources.length]; + private final DeletionTime[] markerTime = new DeletionTime[sources.length]; + + public MergeListener(DecoratedKey partitionKey, PartitionColumns columns, boolean isReversed) + { + this.partitionKey = partitionKey; + this.columns = columns; + this.isReversed = isReversed; + } + + private PartitionUpdate update(int i) + { + PartitionUpdate upd = repairs[i]; + if (upd == null) + { + upd = new PartitionUpdate(command.metadata(), partitionKey, columns, 1); + repairs[i] = upd; + } + return upd; + } + + private Row.Writer currentRow(int i) + { + Row.Writer row = currentRows[i]; + if (row == null) + { + row = currentClustering == Clustering.STATIC_CLUSTERING ? update(i).staticWriter() : update(i).writer(); + currentClustering.writeTo(row); + currentRows[i] = row; + } + return row; + } + + public void onMergePartitionLevelDeletion(DeletionTime mergedDeletion, DeletionTime[] versions) + { + for (int i = 0; i < versions.length; i++) + { + DeletionTime version = versions[i]; + if (mergedDeletion.supersedes(versions[i])) + update(i).addPartitionDeletion(mergedDeletion); + } + } + + public void onMergingRows(Clustering clustering, + LivenessInfo mergedInfo, + DeletionTime mergedDeletion, + Row[] versions) + { + currentClustering = clustering; + for (int i = 0; i < versions.length; i++) + { + Row version = versions[i]; + + if (version == null || mergedInfo.supersedes(version.primaryKeyLivenessInfo())) + currentRow(i).writePartitionKeyLivenessInfo(mergedInfo); + + if (version == null || mergedDeletion.supersedes(version.deletion())) + currentRow(i).writeRowDeletion(mergedDeletion); + } + } + + public void onMergedComplexDeletion(ColumnDefinition c, DeletionTime mergedCompositeDeletion, DeletionTime[] versions) + { + currentColumn = c; + for (int i = 0; i < versions.length; i++) + { + DeletionTime version = versions[i] == null ? DeletionTime.LIVE : versions[i]; + if (mergedCompositeDeletion.supersedes(version)) + currentRow(i).writeComplexDeletion(c, mergedCompositeDeletion); + } + } + + public void onMergedCells(Cell mergedCell, Cell[] versions) + { + for (int i = 0; i < versions.length; i++) + { + Cell version = versions[i]; + Cell toAdd = version == null ? mergedCell : Cells.diff(mergedCell, version); + if (toAdd != null) + toAdd.writeTo(currentRow(i)); + } + } + + public void onRowDone() + { + for (int i = 0; i < currentRows.length; i++) + { + if (currentRows[i] != null) + currentRows[i].endOfRow(); + } + Arrays.fill(currentRows, null); + } + + public void onMergedRangeTombstoneMarkers(RangeTombstoneMarker merged, RangeTombstoneMarker[] versions) + { + for (int i = 0; i < versions.length; i++) + { + RangeTombstoneMarker marker = versions[i]; + // Note that boundaries are both close and open, so it's not one or the other + if (merged.isClose(isReversed) && markerOpen[i] != null) + { + Slice.Bound open = markerOpen[i]; + Slice.Bound close = merged.isBoundary() ? ((RangeTombstoneBoundaryMarker)merged).createCorrespondingCloseBound(isReversed).clustering() : merged.clustering(); + update(i).addRangeTombstone(Slice.make(isReversed ? close : open, isReversed ? open : close), markerTime[i]); + } + if (merged.isOpen(isReversed) && (marker == null || merged.openDeletionTime(isReversed).supersedes(marker.openDeletionTime(isReversed)))) + { + markerOpen[i] = merged.isBoundary() ? ((RangeTombstoneBoundaryMarker)merged).createCorrespondingOpenBound(isReversed).clustering() : merged.clustering(); + markerTime[i] = merged.openDeletionTime(isReversed); + } + } + } + + public void close() + { + for (int i = 0; i < repairs.length; i++) + { + if (repairs[i] == null) + continue; + + // use a separate verb here because we don't want these to be get the white glove hint- + // on-timeout behavior that a "real" mutation gets + Tracing.trace("Sending read-repair-mutation to {}", sources[i]); + MessageOut msg = new Mutation(repairs[i]).createMessage(MessagingService.Verb.READ_REPAIR); + repairResults.add(MessagingService.instance().sendRR(msg, sources[i])); + } + } + } + } + + private class ShortReadProtectedIterator extends CountingUnfilteredPartitionIterator + { + private final InetAddress source; + private final DataLimits.Counter postReconciliationCounter; + + private ShortReadProtectedIterator(InetAddress source, UnfilteredPartitionIterator iterator, DataLimits.Counter postReconciliationCounter) + { + super(iterator, command.limits().newCounter(command.nowInSec(), false)); + this.source = source; + this.postReconciliationCounter = postReconciliationCounter; + } + + @Override + public UnfilteredRowIterator next() + { + return new ShortReadProtectedRowIterator(super.next()); + } + + private class ShortReadProtectedRowIterator extends WrappingUnfilteredRowIterator + { + private boolean initialReadIsDone; + private UnfilteredRowIterator shortReadContinuation; + private Clustering lastClustering; + + ShortReadProtectedRowIterator(UnfilteredRowIterator iter) + { + super(iter); + } + + @Override + public boolean hasNext() + { + if (super.hasNext()) + return true; + + initialReadIsDone = true; + + if (shortReadContinuation != null && shortReadContinuation.hasNext()) + return true; + + return checkForShortRead(); + } + + @Override + public Unfiltered next() + { + Unfiltered next = initialReadIsDone ? shortReadContinuation.next() : super.next(); + + if (next.kind() == Unfiltered.Kind.ROW) + lastClustering = ((Row)next).clustering(); + + return next; + } + + @Override + public void close() + { + try + { + super.close(); + } + finally + { + if (shortReadContinuation != null) + shortReadContinuation.close(); + } + } + + private boolean checkForShortRead() + { + assert shortReadContinuation == null || !shortReadContinuation.hasNext(); + + // We have a short read if the node this is the result of has returned the requested number of + // rows for that partition (i.e. it has stopped returning results due to the limit), but some of + // those results haven't made it in the final result post-reconciliation due to other nodes + // tombstones. If that is the case, then the node might have more results that we should fetch + // as otherwise we might return less results than required, or results that shouldn't be returned + // (because the node has tombstone that hides future results from other nodes but that haven't + // been returned due to the limit). + // Also note that we only get here once all the results for this node have been returned, and so + // if the node had returned the requested number but we still get there, it imply some results were + // skipped during reconciliation. + if (!counter.isDoneForPartition()) + return false; + + assert !postReconciliationCounter.isDoneForPartition(); + + // We need to try to query enough additional results to fulfill our query, but because we could still + // get short reads on that additional query, just querying the number of results we miss may not be + // enough. But we know that when this node answered n rows (counter.countedInCurrentPartition), only + // x rows (postReconciliationCounter.countedInCurrentPartition()) made it in the final result. + // So our ratio of live rows to requested rows is x/n, so since we miss n-x rows, we estimate that + // we should request m rows so that m * x/n = n-x, that is m = (n^2/x) - n. + // Also note that it's ok if we retrieve more results that necessary since our top level iterator is a + // counting iterator. + int n = postReconciliationCounter.countedInCurrentPartition(); + int x = counter.countedInCurrentPartition(); + int toQuery = x == 0 + ? n * 2 // We didn't got any answer, so (somewhat randomly) ask for twice as much + : Math.max(((n * n) / x) - n, 1); + + DataLimits retryLimits = command.limits().forShortReadRetry(toQuery); + ClusteringIndexFilter filter = command.clusteringIndexFilter(partitionKey()); + ClusteringIndexFilter retryFilter = lastClustering == null ? filter : filter.forPaging(metadata().comparator, lastClustering, false); + SinglePartitionReadCommand cmd = SinglePartitionReadCommand.create(command.metadata(), + command.nowInSec(), + command.columnFilter(), + command.rowFilter(), + retryLimits, + partitionKey(), + retryFilter); + + shortReadContinuation = doShortReadRetry(cmd); + return shortReadContinuation.hasNext(); + } + + private UnfilteredRowIterator doShortReadRetry(SinglePartitionReadCommand retryCommand) + { + DataResolver resolver = new DataResolver(keyspace, retryCommand, ConsistencyLevel.ONE, 1); + ReadCallback handler = new ReadCallback(resolver, ConsistencyLevel.ONE, retryCommand, Collections.singletonList(source)); + if (StorageProxy.canDoLocalRequest(source)) + StageManager.getStage(Stage.READ).maybeExecuteImmediately(new StorageProxy.LocalReadRunnable(retryCommand, handler)); + else + MessagingService.instance().sendRRWithFailure(retryCommand.createMessage(), source, handler); + + // We don't call handler.get() because we want to preserve tombstones since we're still in the middle of merging node results. + handler.awaitResults(); + assert resolver.responses.size() == 1; + return UnfilteredPartitionIterators.getOnlyElement(resolver.responses.get(0).payload.makeIterator(), retryCommand); + } + } + } + + public boolean isDataPresent() + { + return !responses.isEmpty(); + } +} diff --git a/src/java/org/apache/cassandra/service/RowDigestResolver.java b/src/java/org/apache/cassandra/service/DigestResolver.java similarity index 54% rename from src/java/org/apache/cassandra/service/RowDigestResolver.java rename to src/java/org/apache/cassandra/service/DigestResolver.java index 82ccc1a2e2..12b0626e34 100644 --- a/src/java/org/apache/cassandra/service/RowDigestResolver.java +++ b/src/java/org/apache/cassandra/service/DigestResolver.java @@ -20,88 +20,79 @@ package org.apache.cassandra.service; import java.nio.ByteBuffer; import java.util.concurrent.TimeUnit; -import org.apache.cassandra.db.ColumnFamily; -import org.apache.cassandra.db.ReadResponse; -import org.apache.cassandra.db.Row; +import org.apache.cassandra.db.*; +import org.apache.cassandra.db.partitions.PartitionIterator; +import org.apache.cassandra.db.partitions.UnfilteredPartitionIterators; import org.apache.cassandra.net.MessageIn; -public class RowDigestResolver extends AbstractRowResolver +public class DigestResolver extends ResponseResolver { - public RowDigestResolver(String keyspaceName, ByteBuffer key, int maxResponseCount) + private volatile ReadResponse dataResponse; + + public DigestResolver(Keyspace keyspace, ReadCommand command, ConsistencyLevel consistency, int maxResponseCount) { - super(key, keyspaceName, maxResponseCount); + super(keyspace, command, consistency, maxResponseCount); + } + + @Override + public void preprocess(MessageIn message) + { + super.preprocess(message); + if (dataResponse == null && !message.payload.isDigestQuery()) + dataResponse = message.payload; } /** * Special case of resolve() so that CL.ONE reads never throw DigestMismatchException in the foreground */ - public Row getData() + public PartitionIterator getData() { - for (MessageIn message : replies) - { - ReadResponse result = message.payload; - if (!result.isDigestQuery()) - return result.row(); - } - return null; + assert isDataPresent(); + return UnfilteredPartitionIterators.filter(dataResponse.makeIterator(), command.nowInSec()); } /* * This method handles two different scenarios: * - * a) we're handling the initial read, of data from the closest replica + digests - * from the rest. In this case we check the digests against each other, + * a) we're handling the initial read of data from the closest replica + digests + * from the rest. In this case we check the digests against each other, * throw an exception if there is a mismatch, otherwise return the data row. * * b) we're checking additional digests that arrived after the minimum to handle * the requested ConsistencyLevel, i.e. asynchronous read repair check */ - public Row resolve() throws DigestMismatchException + public PartitionIterator resolve() throws DigestMismatchException { + if (responses.size() == 1) + return getData(); + if (logger.isDebugEnabled()) - logger.debug("resolving {} responses", replies.size()); + logger.debug("resolving {} responses", responses.size()); long start = System.nanoTime(); // validate digests against each other; throw immediately on mismatch. - // also extract the data reply, if any. - ColumnFamily data = null; ByteBuffer digest = null; - - for (MessageIn message : replies) + for (MessageIn message : responses) { ReadResponse response = message.payload; - ByteBuffer newDigest; - if (response.isDigestQuery()) - { - newDigest = response.digest(); - } - else - { - // note that this allows for multiple data replies, post-CASSANDRA-5932 - data = response.row().cf; - newDigest = ColumnFamily.digest(data); - } - + ByteBuffer newDigest = response.digest(); if (digest == null) digest = newDigest; else if (!digest.equals(newDigest)) - throw new DigestMismatchException(key, digest, newDigest); + // rely on the fact that only single partition queries use digests + throw new DigestMismatchException(((SinglePartitionReadCommand)command).partitionKey(), digest, newDigest); } if (logger.isDebugEnabled()) logger.debug("resolve: {} ms.", TimeUnit.NANOSECONDS.toMillis(System.nanoTime() - start)); - return new Row(key, data); + + return UnfilteredPartitionIterators.filter(dataResponse.makeIterator(), command.nowInSec()); } public boolean isDataPresent() { - for (MessageIn message : replies) - { - if (!message.payload.isDigestQuery()) - return true; - } - return false; + return dataResponse != null; } } diff --git a/src/java/org/apache/cassandra/service/IResponseResolver.java b/src/java/org/apache/cassandra/service/IResponseResolver.java deleted file mode 100644 index 17c8bff5d3..0000000000 --- a/src/java/org/apache/cassandra/service/IResponseResolver.java +++ /dev/null @@ -1,43 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.apache.cassandra.service; - -import org.apache.cassandra.net.MessageIn; - -public interface IResponseResolver { - - /** - * This Method resolves the responses that are passed in . for example : if - * its write response then all we get is true or false return values which - * implies if the writes were successful but for reads its more complicated - * you need to look at the responses and then based on differences schedule - * repairs . Hence you need to derive a response resolver based on your - * needs from this interface. - */ - public TResolved resolve() throws DigestMismatchException; - - public boolean isDataPresent(); - - /** - * returns the data response without comparing with any digests - */ - public TResolved getData(); - - public void preprocess(MessageIn message); - public Iterable> getMessages(); -} diff --git a/src/java/org/apache/cassandra/service/RangeSliceResponseResolver.java b/src/java/org/apache/cassandra/service/RangeSliceResponseResolver.java deleted file mode 100644 index 640681b148..0000000000 --- a/src/java/org/apache/cassandra/service/RangeSliceResponseResolver.java +++ /dev/null @@ -1,168 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.apache.cassandra.service; - -import java.net.InetAddress; -import java.util.*; -import java.util.concurrent.ConcurrentLinkedQueue; - -import com.google.common.collect.AbstractIterator; - -import org.apache.cassandra.db.ColumnFamily; -import org.apache.cassandra.db.DecoratedKey; -import org.apache.cassandra.db.RangeSliceReply; -import org.apache.cassandra.db.Row; -import org.apache.cassandra.net.AsyncOneResponse; -import org.apache.cassandra.net.MessageIn; -import org.apache.cassandra.utils.CloseableIterator; -import org.apache.cassandra.utils.MergeIterator; -import org.apache.cassandra.utils.Pair; - -/** - * Turns RangeSliceReply objects into row (string -> CF) maps, resolving - * to the most recent ColumnFamily and setting up read repairs as necessary. - */ -public class RangeSliceResponseResolver implements IResponseResolver> -{ - private static final Comparator> pairComparator = new Comparator>() - { - public int compare(Pair o1, Pair o2) - { - return o1.left.key.compareTo(o2.left.key); - } - }; - - private final String keyspaceName; - private final long timestamp; - private List sources; - protected final Collection> responses = new ConcurrentLinkedQueue>(); - public final List repairResults = new ArrayList(); - - public RangeSliceResponseResolver(String keyspaceName, long timestamp) - { - this.keyspaceName = keyspaceName; - this.timestamp = timestamp; - } - - public void setSources(List endpoints) - { - this.sources = endpoints; - } - - public List getData() - { - MessageIn response = responses.iterator().next(); - return response.payload.rows; - } - - // Note: this would deserialize the response a 2nd time if getData was called first. - // (this is not currently an issue since we don't do read repair for range queries.) - public Iterable resolve() - { - ArrayList iters = new ArrayList(responses.size()); - int n = 0; - for (MessageIn response : responses) - { - RangeSliceReply reply = response.payload; - n = Math.max(n, reply.rows.size()); - iters.add(new RowIterator(reply.rows.iterator(), response.from)); - } - // for each row, compute the combination of all different versions seen, and repair incomplete versions - // TODO do we need to call close? - CloseableIterator iter = MergeIterator.get(iters, pairComparator, new Reducer()); - - List resolvedRows = new ArrayList(n); - while (iter.hasNext()) - resolvedRows.add(iter.next()); - - return resolvedRows; - } - - public void preprocess(MessageIn message) - { - responses.add(message); - } - - public boolean isDataPresent() - { - return !responses.isEmpty(); - } - - private static class RowIterator extends AbstractIterator> implements CloseableIterator> - { - private final Iterator iter; - private final InetAddress source; - - private RowIterator(Iterator iter, InetAddress source) - { - this.iter = iter; - this.source = source; - } - - protected Pair computeNext() - { - return iter.hasNext() ? Pair.create(iter.next(), source) : endOfData(); - } - - public void close() {} - } - - public Iterable> getMessages() - { - return responses; - } - - private class Reducer extends MergeIterator.Reducer, Row> - { - List versions = new ArrayList(sources.size()); - List versionSources = new ArrayList(sources.size()); - DecoratedKey key; - - public void reduce(Pair current) - { - key = current.left.key; - versions.add(current.left.cf); - versionSources.add(current.right); - } - - protected Row getReduced() - { - ColumnFamily resolved = versions.size() > 1 - ? RowDataResolver.resolveSuperset(versions, timestamp) - : versions.get(0); - if (versions.size() < sources.size()) - { - // add placeholder rows for sources that didn't have any data, so maybeScheduleRepairs sees them - for (InetAddress source : sources) - { - if (!versionSources.contains(source)) - { - versions.add(null); - versionSources.add(source); - } - } - } - // resolved can be null even if versions doesn't have all nulls because of the call to removeDeleted in resolveSuperSet - if (resolved != null) - repairResults.addAll(RowDataResolver.scheduleRepairs(resolved, keyspaceName, key, versions, versionSources)); - versions.clear(); - versionSources.clear(); - return new Row(key, resolved); - } - } -} diff --git a/src/java/org/apache/cassandra/service/RangeSliceVerbHandler.java b/src/java/org/apache/cassandra/service/RangeSliceVerbHandler.java deleted file mode 100644 index 0f3726cd67..0000000000 --- a/src/java/org/apache/cassandra/service/RangeSliceVerbHandler.java +++ /dev/null @@ -1,40 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.apache.cassandra.service; - -import org.apache.cassandra.db.AbstractRangeCommand; -import org.apache.cassandra.db.RangeSliceReply; -import org.apache.cassandra.net.IVerbHandler; -import org.apache.cassandra.net.MessageIn; -import org.apache.cassandra.net.MessagingService; -import org.apache.cassandra.tracing.Tracing; - -public class RangeSliceVerbHandler implements IVerbHandler -{ - public void doVerb(MessageIn message, int id) - { - if (StorageService.instance.isBootstrapMode()) - { - /* Don't service reads! */ - throw new RuntimeException("Cannot service reads while bootstrapping!"); - } - RangeSliceReply reply = new RangeSliceReply(message.payload.executeLocally()); - Tracing.trace("Enqueuing response to {}", message.from); - MessagingService.instance().sendReply(reply.createMessage(), id, message.from); - } -} diff --git a/src/java/org/apache/cassandra/service/ReadCallback.java b/src/java/org/apache/cassandra/service/ReadCallback.java index 0c008e78e6..d548019ffa 100644 --- a/src/java/org/apache/cassandra/service/ReadCallback.java +++ b/src/java/org/apache/cassandra/service/ReadCallback.java @@ -30,8 +30,8 @@ import org.slf4j.LoggerFactory; import org.apache.cassandra.concurrent.Stage; import org.apache.cassandra.concurrent.StageManager; import org.apache.cassandra.config.DatabaseDescriptor; -import org.apache.cassandra.db.ReadCommand; -import org.apache.cassandra.db.Keyspace; +import org.apache.cassandra.db.*; +import org.apache.cassandra.db.partitions.PartitionIterator; import org.apache.cassandra.exceptions.ReadFailureException; import org.apache.cassandra.exceptions.ReadTimeoutException; import org.apache.cassandra.exceptions.UnavailableException; @@ -46,16 +46,16 @@ import org.apache.cassandra.tracing.Tracing; import org.apache.cassandra.utils.FBUtilities; import org.apache.cassandra.utils.concurrent.SimpleCondition; -public class ReadCallback implements IAsyncCallbackWithFailure +public class ReadCallback implements IAsyncCallbackWithFailure { protected static final Logger logger = LoggerFactory.getLogger( ReadCallback.class ); - public final IResponseResolver resolver; + public final ResponseResolver resolver; private final SimpleCondition condition = new SimpleCondition(); - final long start; + private final long start; final int blockfor; final List endpoints; - private final IReadCommand command; + private final ReadCommand command; private final ConsistencyLevel consistencyLevel; private static final AtomicIntegerFieldUpdater recievedUpdater = AtomicIntegerFieldUpdater.newUpdater(ReadCallback.class, "received"); @@ -69,14 +69,17 @@ public class ReadCallback implements IAsyncCallbackWithFail /** * Constructor when response count has to be calculated and blocked for. */ - public ReadCallback(IResponseResolver resolver, ConsistencyLevel consistencyLevel, IReadCommand command, List filteredEndpoints) + public ReadCallback(ResponseResolver resolver, ConsistencyLevel consistencyLevel, ReadCommand command, List filteredEndpoints) { - this(resolver, consistencyLevel, consistencyLevel.blockFor(Keyspace.open(command.getKeyspace())), command, Keyspace.open(command.getKeyspace()), filteredEndpoints); - if (logger.isTraceEnabled()) - logger.trace(String.format("Blockfor is %s; setting up requests to %s", blockfor, StringUtils.join(this.endpoints, ","))); + this(resolver, + consistencyLevel, + consistencyLevel.blockFor(Keyspace.open(command.metadata().ksName)), + command, + Keyspace.open(command.metadata().ksName), + filteredEndpoints); } - public ReadCallback(IResponseResolver resolver, ConsistencyLevel consistencyLevel, int blockfor, IReadCommand command, Keyspace keyspace, List endpoints) + public ReadCallback(ResponseResolver resolver, ConsistencyLevel consistencyLevel, int blockfor, ReadCommand command, Keyspace keyspace, List endpoints) { this.command = command; this.keyspace = keyspace; @@ -86,7 +89,10 @@ public class ReadCallback implements IAsyncCallbackWithFail this.start = System.nanoTime(); this.endpoints = endpoints; // we don't support read repair (or rapid read protection) for range scans yet (CASSANDRA-6897) - assert !(resolver instanceof RangeSliceResponseResolver) || blockfor >= endpoints.size(); + assert !(command instanceof PartitionRangeReadCommand) || blockfor >= endpoints.size(); + + if (logger.isTraceEnabled()) + logger.trace(String.format("Blockfor is %s; setting up requests to %s", blockfor, StringUtils.join(this.endpoints, ","))); } public boolean await(long timePastStart, TimeUnit unit) @@ -102,31 +108,46 @@ public class ReadCallback implements IAsyncCallbackWithFail } } - public TResolved get() throws ReadFailureException, ReadTimeoutException, DigestMismatchException + public void awaitResults() throws ReadFailureException, ReadTimeoutException { - if (!await(command.getTimeout(), TimeUnit.MILLISECONDS)) + boolean signaled = await(command.getTimeout(), TimeUnit.MILLISECONDS); + boolean failed = blockfor + failures > endpoints.size(); + if (signaled && !failed) + return; + + if (Tracing.isTracing()) { - // Same as for writes, see AbstractWriteResponseHandler - ReadTimeoutException ex = new ReadTimeoutException(consistencyLevel, received, blockfor, resolver.isDataPresent()); - Tracing.trace("Read timeout: {}", ex.toString()); - if (logger.isDebugEnabled()) - logger.debug("Read timeout: {}", ex.toString()); - throw ex; + String gotData = received > 0 ? (resolver.isDataPresent() ? " (including data)" : " (only digests)") : ""; + Tracing.trace("{}; received {} of {} responses{}", new Object[]{ (failed ? "Failed" : "Timed out"), received, blockfor, gotData }); + } + else if (logger.isDebugEnabled()) + { + String gotData = received > 0 ? (resolver.isDataPresent() ? " (including data)" : " (only digests)") : ""; + logger.debug("{}; received {} of {} responses{}", new Object[]{ (failed ? "Failed" : "Timed out"), received, blockfor, gotData }); } - if (blockfor + failures > endpoints.size()) - { - ReadFailureException ex = new ReadFailureException(consistencyLevel, received, failures, blockfor, resolver.isDataPresent()); - - if (logger.isDebugEnabled()) - logger.debug("Read failure: {}", ex.toString()); - throw ex; - } - - return blockfor == 1 ? resolver.getData() : resolver.resolve(); + // Same as for writes, see AbstractWriteResponseHandler + throw failed + ? new ReadFailureException(consistencyLevel, received, failures, blockfor, resolver.isDataPresent()) + : new ReadTimeoutException(consistencyLevel, received, blockfor, resolver.isDataPresent()); } - public void response(MessageIn message) + public PartitionIterator get() throws ReadFailureException, ReadTimeoutException, DigestMismatchException + { + awaitResults(); + + PartitionIterator result = blockfor == 1 ? resolver.getData() : resolver.resolve(); + if (logger.isDebugEnabled()) + logger.debug("Read: {} ms.", TimeUnit.NANOSECONDS.toMillis(System.nanoTime() - start)); + return result; + } + + public int blockFor() + { + return blockfor; + } + + public void response(MessageIn message) { resolver.preprocess(message); int n = waitingFor(message.from) @@ -165,13 +186,13 @@ public class ReadCallback implements IAsyncCallbackWithFail return received; } - public void response(TMessage result) + public void response(ReadResponse result) { - MessageIn message = MessageIn.create(FBUtilities.getBroadcastAddress(), - result, - Collections.emptyMap(), - MessagingService.Verb.INTERNAL_RESPONSE, - MessagingService.current_version); + MessageIn message = MessageIn.create(FBUtilities.getBroadcastAddress(), + result, + Collections.emptyMap(), + MessagingService.Verb.INTERNAL_RESPONSE, + MessagingService.current_version); response(message); } @@ -196,7 +217,7 @@ public class ReadCallback implements IAsyncCallbackWithFail public void run() { - // If the resolver is a RowDigestResolver, we need to do a full data read if there is a mismatch. + // If the resolver is a DigestResolver, we need to do a full data read if there is a mismatch. // Otherwise, resolve will send the repairs directly if needs be (and in that case we should never // get a digest mismatch) try @@ -205,7 +226,7 @@ public class ReadCallback implements IAsyncCallbackWithFail } catch (DigestMismatchException e) { - assert resolver instanceof RowDigestResolver; + assert resolver instanceof DigestResolver; if (traceState != null) traceState.trace("Digest mismatch: {}", e.toString()); @@ -214,11 +235,10 @@ public class ReadCallback implements IAsyncCallbackWithFail ReadRepairMetrics.repairedBackground.mark(); - ReadCommand readCommand = (ReadCommand) command; - final RowDataResolver repairResolver = new RowDataResolver(readCommand.ksName, readCommand.key, readCommand.filter(), readCommand.timestamp, endpoints.size()); + final DataResolver repairResolver = new DataResolver(keyspace, command, consistencyLevel, endpoints.size()); AsyncRepairCallback repairHandler = new AsyncRepairCallback(repairResolver, endpoints.size()); - MessageOut message = ((ReadCommand) command).createMessage(); + MessageOut message = command.createMessage(); for (InetAddress endpoint : endpoints) MessagingService.instance().sendRR(message, endpoint, repairHandler); } diff --git a/src/java/org/apache/cassandra/service/AbstractRowResolver.java b/src/java/org/apache/cassandra/service/ResponseResolver.java similarity index 60% rename from src/java/org/apache/cassandra/service/AbstractRowResolver.java rename to src/java/org/apache/cassandra/service/ResponseResolver.java index f3620479d2..e7c94a1b94 100644 --- a/src/java/org/apache/cassandra/service/AbstractRowResolver.java +++ b/src/java/org/apache/cassandra/service/ResponseResolver.java @@ -17,40 +17,45 @@ */ package org.apache.cassandra.service; -import java.nio.ByteBuffer; - import org.slf4j.Logger; import org.slf4j.LoggerFactory; -import org.apache.cassandra.db.DecoratedKey; -import org.apache.cassandra.db.ReadResponse; -import org.apache.cassandra.db.Row; +import org.apache.cassandra.db.*; +import org.apache.cassandra.db.partitions.*; import org.apache.cassandra.net.MessageIn; import org.apache.cassandra.utils.concurrent.Accumulator; -public abstract class AbstractRowResolver implements IResponseResolver +public abstract class ResponseResolver { - protected static final Logger logger = LoggerFactory.getLogger(AbstractRowResolver.class); + protected static final Logger logger = LoggerFactory.getLogger(ResponseResolver.class); + + protected final Keyspace keyspace; + protected final ReadCommand command; + protected final ConsistencyLevel consistency; - protected final String keyspaceName; // Accumulator gives us non-blocking thread-safety with optimal algorithmic constraints - protected final Accumulator> replies; - protected final DecoratedKey key; + protected final Accumulator> responses; - public AbstractRowResolver(ByteBuffer key, String keyspaceName, int maxResponseCount) + public ResponseResolver(Keyspace keyspace, ReadCommand command, ConsistencyLevel consistency, int maxResponseCount) { - this.key = StorageService.getPartitioner().decorateKey(key); - this.keyspaceName = keyspaceName; - this.replies = new Accumulator<>(maxResponseCount); + this.keyspace = keyspace; + this.command = command; + this.consistency = consistency; + this.responses = new Accumulator<>(maxResponseCount); } + public abstract PartitionIterator getData(); + public abstract PartitionIterator resolve() throws DigestMismatchException; + + public abstract boolean isDataPresent(); + public void preprocess(MessageIn message) { - replies.add(message); + responses.add(message); } public Iterable> getMessages() { - return replies; + return responses; } } diff --git a/src/java/org/apache/cassandra/service/RowDataResolver.java b/src/java/org/apache/cassandra/service/RowDataResolver.java deleted file mode 100644 index e935ce7aed..0000000000 --- a/src/java/org/apache/cassandra/service/RowDataResolver.java +++ /dev/null @@ -1,177 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.apache.cassandra.service; - -import java.net.InetAddress; -import java.nio.ByteBuffer; -import java.util.ArrayList; -import java.util.Collections; -import java.util.List; -import java.util.concurrent.TimeUnit; - -import com.google.common.collect.Iterables; - -import org.apache.cassandra.db.*; -import org.apache.cassandra.db.columniterator.IdentityQueryFilter; -import org.apache.cassandra.db.filter.IDiskAtomFilter; -import org.apache.cassandra.db.filter.QueryFilter; -import org.apache.cassandra.net.*; -import org.apache.cassandra.tracing.Tracing; -import org.apache.cassandra.utils.CloseableIterator; -import org.apache.cassandra.utils.FBUtilities; - -public class RowDataResolver extends AbstractRowResolver -{ - private int maxLiveCount = 0; - public List repairResults = Collections.emptyList(); - private final IDiskAtomFilter filter; - private final long timestamp; - - public RowDataResolver(String keyspaceName, ByteBuffer key, IDiskAtomFilter qFilter, long timestamp, int maxResponseCount) - { - super(key, keyspaceName, maxResponseCount); - this.filter = qFilter; - this.timestamp = timestamp; - } - - /* - * This method handles the following scenario: - * - * there was a mismatch on the initial read, so we redid the digest requests - * as full data reads. In this case we need to compute the most recent version - * of each column, and send diffs to out-of-date replicas. - */ - public Row resolve() throws DigestMismatchException - { - int replyCount = replies.size(); - if (logger.isDebugEnabled()) - logger.debug("resolving {} responses", replyCount); - long start = System.nanoTime(); - - ColumnFamily resolved; - if (replyCount > 1) - { - List versions = new ArrayList<>(replyCount); - List endpoints = new ArrayList<>(replyCount); - - for (MessageIn message : replies) - { - ReadResponse response = message.payload; - ColumnFamily cf = response.row().cf; - assert !response.isDigestQuery() : "Received digest response to repair read from " + message.from; - versions.add(cf); - endpoints.add(message.from); - - // compute maxLiveCount to prevent short reads -- see https://issues.apache.org/jira/browse/CASSANDRA-2643 - int liveCount = cf == null ? 0 : filter.getLiveCount(cf, timestamp); - if (liveCount > maxLiveCount) - maxLiveCount = liveCount; - } - - resolved = resolveSuperset(versions, timestamp); - if (logger.isDebugEnabled()) - logger.debug("versions merged"); - - // send updates to any replica that was missing part of the full row - // (resolved can be null even if versions doesn't have all nulls because of the call to removeDeleted in resolveSuperSet) - if (resolved != null) - repairResults = scheduleRepairs(resolved, keyspaceName, key, versions, endpoints); - } - else - { - resolved = replies.get(0).payload.row().cf; - } - - if (logger.isDebugEnabled()) - logger.debug("resolve: {} ms.", TimeUnit.NANOSECONDS.toMillis(System.nanoTime() - start)); - - return new Row(key, resolved); - } - - /** - * For each row version, compare with resolved (the superset of all row versions); - * if it is missing anything, send a mutation to the endpoint it come from. - */ - public static List scheduleRepairs(ColumnFamily resolved, String keyspaceName, DecoratedKey key, List versions, List endpoints) - { - List results = new ArrayList(versions.size()); - - for (int i = 0; i < versions.size(); i++) - { - ColumnFamily diffCf = ColumnFamily.diff(versions.get(i), resolved); - if (diffCf == null) // no repair needs to happen - continue; - - // create and send the mutation message based on the diff - Mutation mutation = new Mutation(keyspaceName, key.getKey(), diffCf); - // use a separate verb here because we don't want these to be get the white glove hint- - // on-timeout behavior that a "real" mutation gets - Tracing.trace("Sending read-repair-mutation to {}", endpoints.get(i)); - results.add(MessagingService.instance().sendRR(mutation.createMessage(MessagingService.Verb.READ_REPAIR), - endpoints.get(i))); - } - - return results; - } - - static ColumnFamily resolveSuperset(Iterable versions, long now) - { - assert Iterables.size(versions) > 0; - - ColumnFamily resolved = null; - for (ColumnFamily cf : versions) - { - if (cf == null) - continue; - - if (resolved == null) - resolved = cf.cloneMeShallow(); - else - resolved.delete(cf); - } - if (resolved == null) - return null; - - // mimic the collectCollatedColumn + removeDeleted path that getColumnFamily takes. - // this will handle removing columns and subcolumns that are suppressed by a row or - // supercolumn tombstone. - QueryFilter filter = new QueryFilter(null, resolved.metadata().cfName, new IdentityQueryFilter(), now); - List> iters = new ArrayList<>(Iterables.size(versions)); - for (ColumnFamily version : versions) - if (version != null) - iters.add(FBUtilities.closeableIterator(version.iterator())); - filter.collateColumns(resolved, iters, Integer.MIN_VALUE); - return ColumnFamilyStore.removeDeleted(resolved, Integer.MIN_VALUE); - } - - public Row getData() - { - assert !replies.isEmpty(); - return replies.get(0).payload.row(); - } - - public boolean isDataPresent() - { - return !replies.isEmpty(); - } - - public int getMaxLiveCount() - { - return maxLiveCount; - } -} diff --git a/src/java/org/apache/cassandra/service/StorageProxy.java b/src/java/org/apache/cassandra/service/StorageProxy.java index 032765ada4..17a9c11976 100644 --- a/src/java/org/apache/cassandra/service/StorageProxy.java +++ b/src/java/org/apache/cassandra/service/StorageProxy.java @@ -41,8 +41,9 @@ import org.apache.cassandra.config.CFMetaData; import org.apache.cassandra.config.DatabaseDescriptor; import org.apache.cassandra.config.Schema; import org.apache.cassandra.db.*; +import org.apache.cassandra.db.rows.*; +import org.apache.cassandra.db.partitions.*; import org.apache.cassandra.db.filter.TombstoneOverwhelmingException; -import org.apache.cassandra.db.index.SecondaryIndex; import org.apache.cassandra.db.index.SecondaryIndexSearcher; import org.apache.cassandra.db.marshal.UUIDType; import org.apache.cassandra.dht.AbstractBounds; @@ -195,13 +196,13 @@ public class StorageProxy implements StorageProxyMBean * @return null if the operation succeeds in updating the row, or the current values corresponding to conditions. * (since, if the CAS doesn't succeed, it means the current value do not match the conditions). */ - public static ColumnFamily cas(String keyspaceName, - String cfName, - ByteBuffer key, - CASRequest request, - ConsistencyLevel consistencyForPaxos, - ConsistencyLevel consistencyForCommit, - ClientState state) + public static RowIterator cas(String keyspaceName, + String cfName, + DecoratedKey key, + CASRequest request, + ConsistencyLevel consistencyForPaxos, + ConsistencyLevel consistencyForCommit, + ClientState state) throws UnavailableException, IsBootstrappingException, RequestFailureException, RequestTimeoutException, InvalidRequestException { final long start = System.nanoTime(); @@ -217,32 +218,35 @@ public class StorageProxy implements StorageProxyMBean while (System.nanoTime() - start < timeout) { // for simplicity, we'll do a single liveness check at the start of each attempt - Pair, Integer> p = getPaxosParticipants(keyspaceName, key, consistencyForPaxos); + Pair, Integer> p = getPaxosParticipants(metadata, key, consistencyForPaxos); List liveEndpoints = p.left; int requiredParticipants = p.right; final Pair pair = beginAndRepairPaxos(start, key, metadata, liveEndpoints, requiredParticipants, consistencyForPaxos, consistencyForCommit, true, state); final UUID ballot = pair.left; contentions += pair.right; + // read the current values and check they validate the conditions Tracing.trace("Reading existing values for CAS precondition"); - long timestamp = System.currentTimeMillis(); - ReadCommand readCommand = ReadCommand.create(keyspaceName, key, cfName, timestamp, request.readFilter()); - List rows = read(Arrays.asList(readCommand), consistencyForPaxos == ConsistencyLevel.LOCAL_SERIAL - ? ConsistencyLevel.LOCAL_QUORUM - : ConsistencyLevel.QUORUM); - ColumnFamily current = rows.get(0).cf; + SinglePartitionReadCommand readCommand = request.readCommand(FBUtilities.nowInSeconds()); + ConsistencyLevel readConsistency = consistencyForPaxos == ConsistencyLevel.LOCAL_SERIAL ? ConsistencyLevel.LOCAL_QUORUM : ConsistencyLevel.QUORUM; + + FilteredPartition current; + try (RowIterator rowIter = readOne(readCommand, readConsistency)) + { + current = FilteredPartition.create(rowIter); + } + if (!request.appliesTo(current)) { Tracing.trace("CAS precondition does not match current values {}", current); - // We should not return null as this means success casWriteMetrics.conditionNotMet.inc(); - return current == null ? ArrayBackedSortedColumns.factory.create(metadata) : current; + return current.rowIterator(); } // finish the paxos round w/ the desired updates // TODO turn null updates into delete? - ColumnFamily updates = request.makeUpdates(current); + PartitionUpdate updates = request.makeUpdates(current); // Apply triggers to cas updates. A consideration here is that // triggers emit Mutations, and so a given trigger implementation @@ -251,9 +255,10 @@ public class StorageProxy implements StorageProxyMBean // validate that the generated mutations are targetted at the same // partition as the initial updates and reject (via an // InvalidRequestException) any which aren't. - updates = TriggerExecutor.instance.execute(key, updates); + updates = TriggerExecutor.instance.execute(updates); - Commit proposal = Commit.newProposal(key, ballot, updates); + + Commit proposal = Commit.newProposal(ballot, updates); Tracing.trace("CAS precondition is met; proposing client-requested updates for {}", ballot); if (proposePaxos(proposal, liveEndpoints, requiredParticipants, true, consistencyForPaxos)) { @@ -305,12 +310,11 @@ public class StorageProxy implements StorageProxyMBean }; } - private static Pair, Integer> getPaxosParticipants(String keyspaceName, ByteBuffer key, ConsistencyLevel consistencyForPaxos) throws UnavailableException + private static Pair, Integer> getPaxosParticipants(CFMetaData cfm, DecoratedKey key, ConsistencyLevel consistencyForPaxos) throws UnavailableException { - Token tk = StorageService.getPartitioner().getToken(key); - List naturalEndpoints = StorageService.instance.getNaturalEndpoints(keyspaceName, tk); - Collection pendingEndpoints = StorageService.instance.getTokenMetadata().pendingEndpointsFor(tk, keyspaceName); - + Token tk = key.getToken(); + List naturalEndpoints = StorageService.instance.getNaturalEndpoints(cfm.ksName, tk); + Collection pendingEndpoints = StorageService.instance.getTokenMetadata().pendingEndpointsFor(tk, cfm.ksName); if (consistencyForPaxos == ConsistencyLevel.LOCAL_SERIAL) { // Restrict naturalEndpoints and pendingEndpoints to node in the local DC only @@ -344,7 +348,7 @@ public class StorageProxy implements StorageProxyMBean * nodes have seen the mostRecentCommit. Otherwise, return null. */ private static Pair beginAndRepairPaxos(long start, - ByteBuffer key, + DecoratedKey key, CFMetaData metadata, List liveEndpoints, int requiredParticipants, @@ -393,7 +397,7 @@ public class StorageProxy implements StorageProxyMBean casWriteMetrics.unfinishedCommit.inc(); else casReadMetrics.unfinishedCommit.inc(); - Commit refreshedInProgress = Commit.newProposal(inProgress.key, ballot, inProgress.update); + Commit refreshedInProgress = Commit.newProposal(ballot, inProgress.update); if (proposePaxos(refreshedInProgress, liveEndpoints, requiredParticipants, false, consistencyForPaxos)) { try @@ -451,7 +455,7 @@ public class StorageProxy implements StorageProxyMBean private static PrepareCallback preparePaxos(Commit toPrepare, List endpoints, int requiredParticipants, ConsistencyLevel consistencyForPaxos) throws WriteTimeoutException { - PrepareCallback callback = new PrepareCallback(toPrepare.key, toPrepare.update.metadata(), requiredParticipants, consistencyForPaxos); + PrepareCallback callback = new PrepareCallback(toPrepare.update.partitionKey(), toPrepare.update.metadata(), requiredParticipants, consistencyForPaxos); MessageOut message = new MessageOut(MessagingService.Verb.PAXOS_PREPARE, toPrepare, Commit.serializer); for (InetAddress target : endpoints) MessagingService.instance().sendRR(message, target, callback); @@ -483,7 +487,7 @@ public class StorageProxy implements StorageProxyMBean boolean shouldBlock = consistencyLevel != ConsistencyLevel.ANY; Keyspace keyspace = Keyspace.open(proposal.update.metadata().ksName); - Token tk = StorageService.getPartitioner().getToken(proposal.key); + Token tk = proposal.update.partitionKey().getToken(); List naturalEndpoints = StorageService.instance.getNaturalEndpoints(keyspace.getName(), tk); Collection pendingEndpoints = StorageService.instance.getTokenMetadata().pendingEndpointsFor(tk, keyspace.getName()); @@ -604,7 +608,7 @@ public class StorageProxy implements StorageProxyMBean if (mutation instanceof CounterMutation) continue; - Token tk = StorageService.getPartitioner().getToken(mutation.key()); + Token tk = mutation.key().getToken(); List naturalEndpoints = StorageService.instance.getNaturalEndpoints(mutation.getKeyspaceName(), tk); Collection pendingEndpoints = StorageService.instance.getTokenMetadata().pendingEndpointsFor(tk, mutation.getKeyspaceName()); for (InetAddress target : Iterables.concat(naturalEndpoints, pendingEndpoints)) @@ -699,6 +703,12 @@ public class StorageProxy implements StorageProxyMBean } } + public static boolean canDoLocalRequest(InetAddress replica) + { + return replica.equals(FBUtilities.getBroadcastAddress()) && OPTIMIZE_LOCAL_REQUESTS; + } + + private static void syncWriteToBatchlog(Collection mutations, Collection endpoints, UUID uuid) throws WriteTimeoutException, WriteFailureException { @@ -714,7 +724,7 @@ public class StorageProxy implements StorageProxyMBean for (InetAddress target : endpoints) { int targetVersion = MessagingService.instance().getVersion(target); - if (target.equals(FBUtilities.getBroadcastAddress()) && OPTIMIZE_LOCAL_REQUESTS) + if (canDoLocalRequest(target)) { insertLocal(message.payload, handler); } @@ -743,12 +753,12 @@ public class StorageProxy implements StorageProxyMBean Keyspace.open(SystemKeyspace.NAME), null, WriteType.SIMPLE); - Mutation mutation = new Mutation(SystemKeyspace.NAME, UUIDType.instance.decompose(uuid)); - mutation.delete(SystemKeyspace.BATCHLOG, FBUtilities.timestampMicros()); + Mutation mutation = new Mutation(SystemKeyspace.NAME, StorageService.getPartitioner().decorateKey(UUIDType.instance.decompose(uuid))); + mutation.add(PartitionUpdate.fullPartitionDelete(SystemKeyspace.Batchlog, mutation.key(), FBUtilities.timestampMicros(), FBUtilities.nowInSeconds())); MessageOut message = mutation.createMessage(); for (InetAddress target : endpoints) { - if (target.equals(FBUtilities.getBroadcastAddress()) && OPTIMIZE_LOCAL_REQUESTS) + if (canDoLocalRequest(target)) insertLocal(message.payload, handler); else MessagingService.instance().sendRR(message, target, handler, false); @@ -793,7 +803,7 @@ public class StorageProxy implements StorageProxyMBean String keyspaceName = mutation.getKeyspaceName(); AbstractReplicationStrategy rs = Keyspace.open(keyspaceName).getReplicationStrategy(); - Token tk = StorageService.getPartitioner().getToken(mutation.key()); + Token tk = mutation.key().getToken(); List naturalEndpoints = StorageService.instance.getNaturalEndpoints(keyspaceName, tk); Collection pendingEndpoints = StorageService.instance.getTokenMetadata().pendingEndpointsFor(tk, keyspaceName); @@ -811,7 +821,7 @@ public class StorageProxy implements StorageProxyMBean { AbstractReplicationStrategy rs = Keyspace.open(mutation.getKeyspaceName()).getReplicationStrategy(); String keyspaceName = mutation.getKeyspaceName(); - Token tk = StorageService.getPartitioner().getToken(mutation.key()); + Token tk = mutation.key().getToken(); List naturalEndpoints = StorageService.instance.getNaturalEndpoints(keyspaceName, tk); Collection pendingEndpoints = StorageService.instance.getTokenMetadata().pendingEndpointsFor(tk, keyspaceName); AbstractWriteResponseHandler responseHandler = rs.getWriteResponseHandler(naturalEndpoints, pendingEndpoints, consistency_level, null, writeType); @@ -903,7 +913,7 @@ public class StorageProxy implements StorageProxyMBean if (FailureDetector.instance.isAlive(destination)) { - if (destination.equals(FBUtilities.getBroadcastAddress()) && OPTIMIZE_LOCAL_REQUESTS) + if (canDoLocalRequest(destination)) { insertLocal = true; } else @@ -1064,7 +1074,7 @@ public class StorageProxy implements StorageProxyMBean } catch (Exception ex) { - logger.error("Failed to apply mutation locally : {}", ex.getMessage()); + logger.error("Failed to apply mutation locally : {}", ex); responseHandler.onFailure(FBUtilities.getBroadcastAddress()); } } @@ -1098,7 +1108,7 @@ public class StorageProxy implements StorageProxyMBean // Exit now if we can't fulfill the CL here instead of forwarding to the leader replica String keyspaceName = cm.getKeyspaceName(); AbstractReplicationStrategy rs = Keyspace.open(keyspaceName).getReplicationStrategy(); - Token tk = StorageService.getPartitioner().getToken(cm.key()); + Token tk = cm.key().getToken(); List naturalEndpoints = StorageService.instance.getNaturalEndpoints(keyspaceName, tk); Collection pendingEndpoints = StorageService.instance.getTokenMetadata().pendingEndpointsFor(tk, keyspaceName); @@ -1123,7 +1133,7 @@ public class StorageProxy implements StorageProxyMBean * is unclear we want to mix those latencies with read latencies, so this * may be a bit involved. */ - private static InetAddress findSuitableEndpoint(String keyspaceName, ByteBuffer key, String localDataCenter, ConsistencyLevel cl) throws UnavailableException + private static InetAddress findSuitableEndpoint(String keyspaceName, DecoratedKey key, String localDataCenter, ConsistencyLevel cl) throws UnavailableException { Keyspace keyspace = Keyspace.open(keyspaceName); IEndpointSnitch snitch = DatabaseDescriptor.getEndpointSnitch(); @@ -1189,57 +1199,69 @@ public class StorageProxy implements StorageProxyMBean }; } - private static boolean systemKeyspaceQuery(List cmds) + private static boolean systemKeyspaceQuery(List cmds) { for (ReadCommand cmd : cmds) - if (!cmd.ksName.equals(SystemKeyspace.NAME)) + if (!cmd.metadata().ksName.equals(SystemKeyspace.NAME)) return false; return true; } - public static List read(List commands, ConsistencyLevel consistencyLevel) + public static RowIterator readOne(SinglePartitionReadCommand command, ConsistencyLevel consistencyLevel) + throws UnavailableException, IsBootstrappingException, ReadFailureException, ReadTimeoutException, InvalidRequestException + { + return readOne(command, consistencyLevel, null); + } + + public static RowIterator readOne(SinglePartitionReadCommand command, ConsistencyLevel consistencyLevel, ClientState state) + throws UnavailableException, IsBootstrappingException, ReadFailureException, ReadTimeoutException, InvalidRequestException + { + return PartitionIterators.getOnlyElement(read(SinglePartitionReadCommand.Group.one(command), consistencyLevel, state), command); + } + + public static PartitionIterator read(SinglePartitionReadCommand.Group group, ConsistencyLevel consistencyLevel) throws UnavailableException, IsBootstrappingException, ReadFailureException, ReadTimeoutException, InvalidRequestException { // When using serial CL, the ClientState should be provided assert !consistencyLevel.isSerialConsistency(); - return read(commands, consistencyLevel, null); + return read(group, consistencyLevel, null); } /** * Performs the actual reading of a row out of the StorageService, fetching * a specific set of column names from a given column family. */ - public static List read(List commands, ConsistencyLevel consistencyLevel, ClientState state) + public static PartitionIterator read(SinglePartitionReadCommand.Group group, ConsistencyLevel consistencyLevel, ClientState state) throws UnavailableException, IsBootstrappingException, ReadFailureException, ReadTimeoutException, InvalidRequestException { - if (StorageService.instance.isBootstrapMode() && !systemKeyspaceQuery(commands)) + if (StorageService.instance.isBootstrapMode() && !systemKeyspaceQuery(group.commands)) { readMetrics.unavailables.mark(); throw new IsBootstrappingException(); } return consistencyLevel.isSerialConsistency() - ? readWithPaxos(commands, consistencyLevel, state) - : readRegular(commands, consistencyLevel); + ? readWithPaxos(group, consistencyLevel, state) + : readRegular(group, consistencyLevel); } - private static List readWithPaxos(List commands, ConsistencyLevel consistencyLevel, ClientState state) + private static PartitionIterator readWithPaxos(SinglePartitionReadCommand.Group group, ConsistencyLevel consistencyLevel, ClientState state) throws InvalidRequestException, UnavailableException, ReadFailureException, ReadTimeoutException { assert state != null; + if (group.commands.size() > 1) + throw new InvalidRequestException("SERIAL/LOCAL_SERIAL consistency may only be requested for one partition at a time"); long start = System.nanoTime(); - List rows = null; + SinglePartitionReadCommand command = group.commands.get(0); + CFMetaData metadata = command.metadata(); + DecoratedKey key = command.partitionKey(); + PartitionIterator result = null; try { // make sure any in-progress paxos writes are done (i.e., committed to a majority of replicas), before performing a quorum read - if (commands.size() > 1) - throw new InvalidRequestException("SERIAL/LOCAL_SERIAL consistency may only be requested for one row at a time"); - ReadCommand command = commands.get(0); - - CFMetaData metadata = Schema.instance.getCFMetaData(command.ksName, command.cfName); - Pair, Integer> p = getPaxosParticipants(command.ksName, command.key, consistencyLevel); + Pair, Integer> p = getPaxosParticipants(metadata, key, consistencyLevel); List liveEndpoints = p.left; int requiredParticipants = p.right; @@ -1247,22 +1269,23 @@ public class StorageProxy implements StorageProxyMBean final ConsistencyLevel consistencyForCommitOrFetch = consistencyLevel == ConsistencyLevel.LOCAL_SERIAL ? ConsistencyLevel.LOCAL_QUORUM : ConsistencyLevel.QUORUM; + try { - final Pair pair = beginAndRepairPaxos(start, command.key, metadata, liveEndpoints, requiredParticipants, consistencyLevel, consistencyForCommitOrFetch, false, state); + final Pair pair = beginAndRepairPaxos(start, key, metadata, liveEndpoints, requiredParticipants, consistencyLevel, consistencyForCommitOrFetch, false, state); if (pair.right > 0) casReadMetrics.contention.update(pair.right); } catch (WriteTimeoutException e) { - throw new ReadTimeoutException(consistencyLevel, 0, consistencyLevel.blockFor(Keyspace.open(command.ksName)), false); + throw new ReadTimeoutException(consistencyLevel, 0, consistencyLevel.blockFor(Keyspace.open(metadata.ksName)), false); } catch (WriteFailureException e) { throw new ReadFailureException(consistencyLevel, e.received, e.failures, e.blockFor, false); } - rows = fetchRows(commands, consistencyForCommitOrFetch); + result = fetchRows(group.commands, consistencyForCommitOrFetch); } catch (UnavailableException e) { @@ -1287,23 +1310,25 @@ public class StorageProxy implements StorageProxyMBean long latency = System.nanoTime() - start; readMetrics.addNano(latency); casReadMetrics.addNano(latency); - // TODO avoid giving every command the same latency number. Can fix this in CASSADRA-5329 - for (ReadCommand command : commands) - Keyspace.open(command.ksName).getColumnFamilyStore(command.cfName).metric.coordinatorReadLatency.update(latency, TimeUnit.NANOSECONDS); + Keyspace.open(metadata.ksName).getColumnFamilyStore(metadata.cfName).metric.coordinatorReadLatency.update(latency, TimeUnit.NANOSECONDS); } - return rows; + return result; } - private static List readRegular(List commands, ConsistencyLevel consistencyLevel) + @SuppressWarnings("resource") + private static PartitionIterator readRegular(SinglePartitionReadCommand.Group group, ConsistencyLevel consistencyLevel) throws UnavailableException, ReadFailureException, ReadTimeoutException { long start = System.nanoTime(); - List rows = null; - try { - rows = fetchRows(commands, consistencyLevel); + PartitionIterator result = fetchRows(group.commands, consistencyLevel); + // If we have more than one command, then despite each read command honoring the limit, the total result + // might not honor it and so we should enforce it + if (group.commands.size() > 1) + result = group.limits().filter(result, group.nowInSec()); + return result; } catch (UnavailableException e) { @@ -1325,11 +1350,9 @@ public class StorageProxy implements StorageProxyMBean long latency = System.nanoTime() - start; readMetrics.addNano(latency); // TODO avoid giving every command the same latency number. Can fix this in CASSADRA-5329 - for (ReadCommand command : commands) - Keyspace.open(command.ksName).getColumnFamilyStore(command.cfName).metric.coordinatorReadLatency.update(latency, TimeUnit.NANOSECONDS); + for (ReadCommand command : group.commands) + Keyspace.openAndGetStore(command.metadata()).metric.coordinatorReadLatency.update(latency, TimeUnit.NANOSECONDS); } - - return rows; } /** @@ -1343,182 +1366,142 @@ public class StorageProxy implements StorageProxyMBean * 4. If the digests (if any) match the data return the data * 5. else carry out read repair by getting data from all the nodes. */ - private static List fetchRows(List initialCommands, ConsistencyLevel consistencyLevel) + private static PartitionIterator fetchRows(List> commands, ConsistencyLevel consistencyLevel) throws UnavailableException, ReadFailureException, ReadTimeoutException { - List rows = new ArrayList<>(initialCommands.size()); - // (avoid allocating a new list in the common case of nothing-to-retry) - List commandsToRetry = Collections.emptyList(); + int cmdCount = commands.size(); - do + SinglePartitionReadLifecycle[] reads = new SinglePartitionReadLifecycle[cmdCount]; + for (int i = 0; i < cmdCount; i++) + reads[i] = new SinglePartitionReadLifecycle(commands.get(i), consistencyLevel); + + for (int i = 0; i < cmdCount; i++) + reads[i].doInitialQueries(); + + for (int i = 0; i < cmdCount; i++) + reads[i].maybeTryAdditionalReplicas(); + + for (int i = 0; i < cmdCount; i++) + reads[i].awaitResultsAndRetryOnDigestMismatch(); + + for (int i = 0; i < cmdCount; i++) + if (!reads[i].isDone()) + reads[i].maybeAwaitFullDataRead(); + + List results = new ArrayList<>(cmdCount); + for (int i = 0; i < cmdCount; i++) { - List commands = commandsToRetry.isEmpty() ? initialCommands : commandsToRetry; - AbstractReadExecutor[] readExecutors = new AbstractReadExecutor[commands.size()]; + assert reads[i].isDone(); + results.add(reads[i].getResult()); + } - if (!commandsToRetry.isEmpty()) - Tracing.trace("Retrying {} commands", commandsToRetry.size()); + return PartitionIterators.concat(results); + } - // send out read requests - for (int i = 0; i < commands.size(); i++) + private static class SinglePartitionReadLifecycle + { + private final SinglePartitionReadCommand command; + private final AbstractReadExecutor executor; + private final ConsistencyLevel consistency; + + private PartitionIterator result; + private ReadCallback repairHandler; + + SinglePartitionReadLifecycle(SinglePartitionReadCommand command, ConsistencyLevel consistency) + { + this.command = command; + this.executor = AbstractReadExecutor.getReadExecutor(command, consistency); + this.consistency = consistency; + } + + boolean isDone() + { + return result != null; + } + + void doInitialQueries() + { + executor.executeAsync(); + } + + void maybeTryAdditionalReplicas() + { + executor.maybeTryAdditionalReplicas(); + } + + void awaitResultsAndRetryOnDigestMismatch() throws ReadFailureException, ReadTimeoutException + { + try { - ReadCommand command = commands.get(i); - assert !command.isDigestQuery(); - - AbstractReadExecutor exec = AbstractReadExecutor.getReadExecutor(command, consistencyLevel); - exec.executeAsync(); - readExecutors[i] = exec; + result = executor.get(); } - - for (AbstractReadExecutor exec : readExecutors) - exec.maybeTryAdditionalReplicas(); - - // read results and make a second pass for any digest mismatches - List repairCommands = null; - List> repairResponseHandlers = null; - for (AbstractReadExecutor exec: readExecutors) + catch (DigestMismatchException ex) { - try + Tracing.trace("Digest mismatch: {}", ex); + + ReadRepairMetrics.repairedBlocking.mark(); + + // Do a full data read to resolve the correct response (and repair node that need be) + Keyspace keyspace = Keyspace.open(command.metadata().ksName); + DataResolver resolver = new DataResolver(keyspace, command, ConsistencyLevel.ALL, executor.handler.endpoints.size()); + repairHandler = new ReadCallback(resolver, + ConsistencyLevel.ALL, + executor.getContactedReplicas().size(), + command, + keyspace, + executor.handler.endpoints); + + MessageOut message = command.createMessage(); + for (InetAddress endpoint : executor.getContactedReplicas()) { - Row row = exec.get(); - if (row != null) - { - exec.command.maybeTrim(row); - rows.add(row); - } - - if (logger.isDebugEnabled()) - logger.debug("Read: {} ms.", TimeUnit.NANOSECONDS.toMillis(System.nanoTime() - exec.handler.start)); - } - catch (ReadTimeoutException|ReadFailureException ex) - { - int blockFor = consistencyLevel.blockFor(Keyspace.open(exec.command.getKeyspace())); - int responseCount = exec.handler.getReceivedCount(); - String gotData = responseCount > 0 - ? exec.resolver.isDataPresent() ? " (including data)" : " (only digests)" - : ""; - - boolean isTimeout = ex instanceof ReadTimeoutException; - if (Tracing.isTracing()) - { - Tracing.trace("{}; received {} of {} responses{}", - isTimeout ? "Timed out" : "Failed", responseCount, blockFor, gotData); - } - else if (logger.isDebugEnabled()) - { - logger.debug("Read {}; received {} of {} responses{}", (isTimeout ? "timeout" : "failure"), responseCount, blockFor, gotData); - } - throw ex; - } - catch (DigestMismatchException ex) - { - Tracing.trace("Digest mismatch: {}", ex); - - ReadRepairMetrics.repairedBlocking.mark(); - - // Do a full data read to resolve the correct response (and repair node that need be) - RowDataResolver resolver = new RowDataResolver(exec.command.ksName, exec.command.key, exec.command.filter(), exec.command.timestamp, exec.handler.endpoints.size()); - ReadCallback repairHandler = new ReadCallback<>(resolver, - ConsistencyLevel.ALL, - exec.getContactedReplicas().size(), - exec.command, - Keyspace.open(exec.command.getKeyspace()), - exec.handler.endpoints); - - if (repairCommands == null) - { - repairCommands = new ArrayList<>(); - repairResponseHandlers = new ArrayList<>(); - } - repairCommands.add(exec.command); - repairResponseHandlers.add(repairHandler); - - MessageOut message = exec.command.createMessage(); - for (InetAddress endpoint : exec.getContactedReplicas()) - { - Tracing.trace("Enqueuing full data read to {}", endpoint); - MessagingService.instance().sendRRWithFailure(message, endpoint, repairHandler); - } + Tracing.trace("Enqueuing full data read to {}", endpoint); + MessagingService.instance().sendRRWithFailure(message, endpoint, repairHandler); } } + } - commandsToRetry.clear(); + void maybeAwaitFullDataRead() throws ReadTimeoutException + { + // There wasn't a digest mismatch, we're good + if (repairHandler == null) + return; - // read the results for the digest mismatch retries - if (repairResponseHandlers != null) + // Otherwise, get the result from the full-data read and check that it's not a short read + try { - for (int i = 0; i < repairCommands.size(); i++) - { - ReadCommand command = repairCommands.get(i); - ReadCallback handler = repairResponseHandlers.get(i); - - Row row; - try - { - row = handler.get(); - } - catch (DigestMismatchException e) - { - throw new AssertionError(e); // full data requested from each node here, no digests should be sent - } - catch (ReadTimeoutException e) - { - if (Tracing.isTracing()) - Tracing.trace("Timed out waiting on digest mismatch repair requests"); - else - logger.debug("Timed out waiting on digest mismatch repair requests"); - // the caught exception here will have CL.ALL from the repair command, - // not whatever CL the initial command was at (CASSANDRA-7947) - int blockFor = consistencyLevel.blockFor(Keyspace.open(command.getKeyspace())); - throw new ReadTimeoutException(consistencyLevel, blockFor-1, blockFor, true); - } - - RowDataResolver resolver = (RowDataResolver)handler.resolver; - try - { - // wait for the repair writes to be acknowledged, to minimize impact on any replica that's - // behind on writes in case the out-of-sync row is read multiple times in quick succession - FBUtilities.waitOnFutures(resolver.repairResults, DatabaseDescriptor.getWriteRpcTimeout()); - } - catch (TimeoutException e) - { - if (Tracing.isTracing()) - Tracing.trace("Timed out waiting on digest mismatch repair acknowledgements"); - else - logger.debug("Timed out waiting on digest mismatch repair acknowledgements"); - int blockFor = consistencyLevel.blockFor(Keyspace.open(command.getKeyspace())); - throw new ReadTimeoutException(consistencyLevel, blockFor-1, blockFor, true); - } - - // retry any potential short reads - ReadCommand retryCommand = command.maybeGenerateRetryCommand(resolver, row); - if (retryCommand != null) - { - Tracing.trace("Issuing retry for read command"); - if (commandsToRetry == Collections.EMPTY_LIST) - commandsToRetry = new ArrayList<>(); - commandsToRetry.add(retryCommand); - continue; - } - - if (row != null) - { - command.maybeTrim(row); - rows.add(row); - } - } + result = repairHandler.get(); } - } while (!commandsToRetry.isEmpty()); + catch (DigestMismatchException e) + { + throw new AssertionError(e); // full data requested from each node here, no digests should be sent + } + catch (ReadTimeoutException e) + { + if (Tracing.isTracing()) + Tracing.trace("Timed out waiting on digest mismatch repair requests"); + else + logger.debug("Timed out waiting on digest mismatch repair requests"); + // the caught exception here will have CL.ALL from the repair command, + // not whatever CL the initial command was at (CASSANDRA-7947) + int blockFor = consistency.blockFor(Keyspace.open(command.metadata().ksName)); + throw new ReadTimeoutException(consistency, blockFor-1, blockFor, true); + } + } - return rows; + PartitionIterator getResult() + { + assert result != null; + return result; + } } static class LocalReadRunnable extends DroppableRunnable { private final ReadCommand command; - private final ReadCallback handler; + private final ReadCallback handler; private final long start = System.nanoTime(); - LocalReadRunnable(ReadCommand command, ReadCallback handler) + LocalReadRunnable(ReadCommand command, ReadCallback handler) { super(MessagingService.Verb.READ); this.command = command; @@ -1529,43 +1512,11 @@ public class StorageProxy implements StorageProxyMBean { try { - Keyspace keyspace = Keyspace.open(command.ksName); - Row r = command.getRow(keyspace); - ReadResponse result = ReadVerbHandler.getResponse(command, r); + try (ReadOrderGroup orderGroup = command.startOrderGroup(); UnfilteredPartitionIterator iterator = command.executeLocally(orderGroup)) + { + handler.response(command.createResponse(iterator)); + } MessagingService.instance().addLatency(FBUtilities.getBroadcastAddress(), TimeUnit.NANOSECONDS.toMillis(System.nanoTime() - start)); - handler.response(result); - } - catch (Throwable t) - { - handler.onFailure(FBUtilities.getBroadcastAddress()); - if (t instanceof TombstoneOverwhelmingException) - logger.error(t.getMessage()); - else - throw t; - } - } - } - - static class LocalRangeSliceRunnable extends DroppableRunnable - { - private final AbstractRangeCommand command; - private final ReadCallback> handler; - private final long start = System.nanoTime(); - - LocalRangeSliceRunnable(AbstractRangeCommand command, ReadCallback> handler) - { - super(MessagingService.Verb.RANGE_SLICE); - this.command = command; - this.handler = handler; - } - - protected void runMayThrow() - { - try - { - RangeSliceReply result = new RangeSliceReply(command.executeLocally()); - MessagingService.instance().addLatency(FBUtilities.getBroadcastAddress(), TimeUnit.NANOSECONDS.toMillis(System.nanoTime() - start)); - handler.response(result); } catch (Throwable t) { @@ -1583,7 +1534,7 @@ public class StorageProxy implements StorageProxyMBean return getLiveSortedEndpoints(keyspace, StorageService.getPartitioner().decorateKey(key)); } - private static List getLiveSortedEndpoints(Keyspace keyspace, RingPosition pos) + public static List getLiveSortedEndpoints(Keyspace keyspace, RingPosition pos) { List liveEndpoints = StorageService.instance.getLiveNaturalEndpoints(keyspace, pos); DatabaseDescriptor.getEndpointSnitch().sortByProximity(FBUtilities.getBroadcastAddress(), liveEndpoints); @@ -1603,307 +1554,322 @@ public class StorageProxy implements StorageProxyMBean } /** - * Estimate the number of result rows (either cql3 rows or storage rows, as called for by the command) per + * Estimate the number of result rows (either cql3 rows or "thrift" rows, as called for by the command) per * range in the ring based on our local data. This assumes that ranges are uniformly distributed across the cluster * and that the queried data is also uniformly distributed. */ - private static float estimateResultRowsPerRange(AbstractRangeCommand command, Keyspace keyspace) + private static float estimateResultsPerRange(PartitionRangeReadCommand command, Keyspace keyspace) { - ColumnFamilyStore cfs = keyspace.getColumnFamilyStore(command.columnFamily); - float resultRowsPerRange = Float.POSITIVE_INFINITY; - if (command.rowFilter != null && !command.rowFilter.isEmpty()) + ColumnFamilyStore cfs = keyspace.getColumnFamilyStore(command.metadata().cfId); + SecondaryIndexSearcher searcher = cfs.indexManager.getBestIndexSearcherFor(command); + + float maxExpectedResults = searcher == null + ? command.limits().estimateTotalResults(cfs) + : searcher.highestSelectivityIndex(command.rowFilter()).estimateResultRows(); + + // adjust maxExpectedResults by the number of tokens this node has and the replication factor for this ks + return (maxExpectedResults / DatabaseDescriptor.getNumTokens()) / keyspace.getReplicationStrategy().getReplicationFactor(); + } + + private static class RangeForQuery + { + public final AbstractBounds range; + public final List liveEndpoints; + public final List filteredEndpoints; + + public RangeForQuery(AbstractBounds range, List liveEndpoints, List filteredEndpoints) { - List searchers = cfs.indexManager.getIndexSearchersForQuery(command.rowFilter); - if (searchers.isEmpty()) + this.range = range; + this.liveEndpoints = liveEndpoints; + this.filteredEndpoints = filteredEndpoints; + } + } + + private static class RangeIterator extends AbstractIterator + { + private final Keyspace keyspace; + private final ConsistencyLevel consistency; + private final Iterator> ranges; + private final int rangeCount; + + public RangeIterator(PartitionRangeReadCommand command, Keyspace keyspace, ConsistencyLevel consistency) + { + this.keyspace = keyspace; + this.consistency = consistency; + + List> l = keyspace.getReplicationStrategy() instanceof LocalStrategy + ? command.dataRange().keyRange().unwrap() + : getRestrictedRanges(command.dataRange().keyRange()); + this.ranges = l.iterator(); + this.rangeCount = l.size(); + } + + public int rangeCount() + { + return rangeCount; + } + + protected RangeForQuery computeNext() + { + if (!ranges.hasNext()) + return endOfData(); + + AbstractBounds range = ranges.next(); + List liveEndpoints = getLiveSortedEndpoints(keyspace, range.right); + return new RangeForQuery(range, + liveEndpoints, + consistency.filterForQuery(keyspace, liveEndpoints)); + } + } + + private static class RangeMerger extends AbstractIterator + { + private final Keyspace keyspace; + private final ConsistencyLevel consistency; + private final PeekingIterator ranges; + + private RangeMerger(Iterator iterator, Keyspace keyspace, ConsistencyLevel consistency) + { + this.keyspace = keyspace; + this.consistency = consistency; + this.ranges = Iterators.peekingIterator(iterator); + } + + protected RangeForQuery computeNext() + { + if (!ranges.hasNext()) + return endOfData(); + + RangeForQuery current = ranges.next(); + + // getRestrictedRange has broken the queried range into per-[vnode] token ranges, but this doesn't take + // the replication factor into account. If the intersection of live endpoints for 2 consecutive ranges + // still meets the CL requirements, then we can merge both ranges into the same RangeSliceCommand. + while (ranges.hasNext()) { - resultRowsPerRange = calculateResultRowsUsingEstimatedKeys(cfs); + // If the current range right is the min token, we should stop merging because CFS.getRangeSlice + // don't know how to deal with a wrapping range. + // Note: it would be slightly more efficient to have CFS.getRangeSlice on the destination nodes unwraps + // the range if necessary and deal with it. However, we can't start sending wrapped range without breaking + // wire compatibility, so It's likely easier not to bother; + if (current.range.right.isMinimum()) + break; + + RangeForQuery next = ranges.peek(); + + List merged = intersection(current.liveEndpoints, next.liveEndpoints); + + // Check if there is enough endpoint for the merge to be possible. + if (!consistency.isSufficientLiveNodes(keyspace, merged)) + break; + + List filteredMerged = consistency.filterForQuery(keyspace, merged); + + // Estimate whether merging will be a win or not + if (!DatabaseDescriptor.getEndpointSnitch().isWorthMergingForRangeQuery(filteredMerged, current.filteredEndpoints, next.filteredEndpoints)) + break; + + // If we get there, merge this range and the next one + current = new RangeForQuery(current.range.withNewRight(next.range.right), merged, filteredMerged); + ranges.next(); // consume the range we just merged since we've only peeked so far + } + return current; + } + } + + private static class SingleRangeResponse extends AbstractIterator implements PartitionIterator + { + private final ReadCallback handler; + private PartitionIterator result; + + private SingleRangeResponse(ReadCallback handler) + { + this.handler = handler; + } + + private void waitForResponse() throws ReadTimeoutException + { + if (result != null) + return; + + try + { + result = handler.get(); + } + catch (DigestMismatchException e) + { + throw new AssertionError(e); // no digests in range slices yet + } + } + + protected RowIterator computeNext() + { + waitForResponse(); + return result.hasNext() ? result.next() : endOfData(); + } + + public void close() + { + if (result != null) + result.close(); + } + } + + private static class RangeCommandIterator extends AbstractIterator implements PartitionIterator + { + private final Iterator ranges; + private final int totalRangeCount; + private final PartitionRangeReadCommand command; + private final Keyspace keyspace; + private final ConsistencyLevel consistency; + + private final long startTime; + private CountingPartitionIterator sentQueryIterator; + + private int concurrencyFactor; + // The two following "metric" are maintained to improve the concurrencyFactor + // when it was not good enough initially. + private int liveReturned; + private int rangesQueried; + + public RangeCommandIterator(RangeIterator ranges, PartitionRangeReadCommand command, int concurrencyFactor, Keyspace keyspace, ConsistencyLevel consistency) + { + this.command = command; + this.concurrencyFactor = concurrencyFactor; + this.startTime = System.nanoTime(); + this.ranges = new RangeMerger(ranges, keyspace, consistency); + this.totalRangeCount = ranges.rangeCount(); + this.consistency = consistency; + this.keyspace = keyspace; + } + + public RowIterator computeNext() + { + while (sentQueryIterator == null || !sentQueryIterator.hasNext()) + { + // If we don't have more range to handle, we're done + if (!ranges.hasNext()) + return endOfData(); + + // else, sends the next batch of concurrent queries (after having close the previous iterator) + if (sentQueryIterator != null) + { + liveReturned += sentQueryIterator.counter().counted(); + sentQueryIterator.close(); + + // It's not the first batch of queries and we're not done, so we we can use what has been + // returned so far to improve our rows-per-range estimate and update the concurrency accordingly + updateConcurrencyFactor(); + } + sentQueryIterator = sendNextRequests(); + } + + return sentQueryIterator.next(); + } + + private void updateConcurrencyFactor() + { + if (liveReturned == 0) + { + // we haven't actually gotten any results, so query all remaining ranges at once + concurrencyFactor = totalRangeCount - rangesQueried; + return; + } + + // Otherwise, compute how many rows per range we got on average and pick a concurrency factor + // that should allow us to fetch all remaining rows with the next batch of (concurrent) queries. + int remainingRows = command.limits().count() - liveReturned; + float rowsPerRange = (float)liveReturned / (float)rangesQueried; + concurrencyFactor = Math.max(1, Math.min(totalRangeCount - rangesQueried, Math.round(remainingRows / rowsPerRange))); + logger.debug("Didn't get enough response rows; actual rows per range: {}; remaining rows: {}, new concurrent requests: {}", + rowsPerRange, (int) remainingRows, concurrencyFactor); + } + + private SingleRangeResponse query(RangeForQuery toQuery) + { + PartitionRangeReadCommand rangeCommand = command.forSubRange(toQuery.range); + + DataResolver resolver = new DataResolver(keyspace, rangeCommand, consistency, toQuery.filteredEndpoints.size()); + + int blockFor = consistency.blockFor(keyspace); + int minResponses = Math.min(toQuery.filteredEndpoints.size(), blockFor); + List minimalEndpoints = toQuery.filteredEndpoints.subList(0, minResponses); + ReadCallback handler = new ReadCallback(resolver, consistency, rangeCommand, minimalEndpoints); + + handler.assureSufficientLiveNodes(); + + if (toQuery.filteredEndpoints.size() == 1 && canDoLocalRequest(toQuery.filteredEndpoints.get(0))) + { + StageManager.getStage(Stage.READ).execute(new LocalReadRunnable(rangeCommand, handler), Tracing.instance.get()); } else { - // Secondary index query (cql3 or otherwise). Estimate result rows based on most selective 2ary index. - for (SecondaryIndexSearcher searcher : searchers) + MessageOut message = rangeCommand.createMessage(); + for (InetAddress endpoint : toQuery.filteredEndpoints) { - // use our own mean column count as our estimate for how many matching rows each node will have - SecondaryIndex highestSelectivityIndex = searcher.highestSelectivityIndex(command.rowFilter); - resultRowsPerRange = Math.min(resultRowsPerRange, highestSelectivityIndex.estimateResultRows()); + Tracing.trace("Enqueuing request to {}", endpoint); + MessagingService.instance().sendRRWithFailure(message, endpoint, handler); } } - } - else if (!command.countCQL3Rows()) - { - // non-cql3 query - resultRowsPerRange = cfs.estimateKeys(); - } - else - { - resultRowsPerRange = calculateResultRowsUsingEstimatedKeys(cfs); + + return new SingleRangeResponse(handler); } - // adjust resultRowsPerRange by the number of tokens this node has and the replication factor for this ks - return (resultRowsPerRange / DatabaseDescriptor.getNumTokens()) / keyspace.getReplicationStrategy().getReplicationFactor(); - } + private CountingPartitionIterator sendNextRequests() + { + List concurrentQueries = new ArrayList<>(concurrencyFactor); + for (int i = 0; i < concurrencyFactor && ranges.hasNext(); i++) + { + concurrentQueries.add(query(ranges.next())); + ++rangesQueried; + } - private static float calculateResultRowsUsingEstimatedKeys(ColumnFamilyStore cfs) - { - if (cfs.metadata.comparator.isDense()) - { - // one storage row per result row, so use key estimate directly - return cfs.estimateKeys(); + Tracing.trace("Submitted {} concurrent range requests", concurrentQueries.size()); + return new CountingPartitionIterator(PartitionIterators.concat(concurrentQueries), command.limits(), command.nowInSec()); } - else + + public void close() { - float resultRowsPerStorageRow = ((float) cfs.getMeanColumns()) / cfs.metadata.regularColumns().size(); - return resultRowsPerStorageRow * (cfs.estimateKeys()); + try + { + if (sentQueryIterator != null) + sentQueryIterator.close(); + } + finally + { + long latency = System.nanoTime() - startTime; + rangeMetrics.addNano(latency); + Keyspace.openAndGetStore(command.metadata()).metric.coordinatorScanLatency.update(latency, TimeUnit.NANOSECONDS); + } } } - public static List getRangeSlice(AbstractRangeCommand command, ConsistencyLevel consistency_level) + @SuppressWarnings("resource") + public static PartitionIterator getRangeSlice(PartitionRangeReadCommand command, ConsistencyLevel consistencyLevel) throws UnavailableException, ReadFailureException, ReadTimeoutException { Tracing.trace("Computing ranges to query"); long startTime = System.nanoTime(); - Keyspace keyspace = Keyspace.open(command.keyspace); - List rows; - // now scan until we have enough results - try - { - int liveRowCount = 0; - boolean countLiveRows = command.countCQL3Rows() || command.ignoredTombstonedPartitions(); - rows = new ArrayList<>(); + List partitions = new ArrayList<>(); - // when dealing with LocalStrategy keyspaces, we can skip the range splitting and merging (which can be - // expensive in clusters with vnodes) - List> ranges; - if (keyspace.getReplicationStrategy() instanceof LocalStrategy) - ranges = command.keyRange.unwrap(); - else - ranges = getRestrictedRanges(command.keyRange); + Keyspace keyspace = Keyspace.open(command.metadata().ksName); + RangeIterator ranges = new RangeIterator(command, keyspace, consistencyLevel); - // determine the number of rows to be fetched and the concurrency factor - int rowsToBeFetched = command.limit(); - int concurrencyFactor; - if (command.requiresScanningAllRanges()) - { - // all nodes must be queried - rowsToBeFetched *= ranges.size(); - concurrencyFactor = ranges.size(); - logger.debug("Requested rows: {}, ranges.size(): {}; concurrent range requests: {}", - command.limit(), - ranges.size(), - concurrencyFactor); - Tracing.trace("Submitting range requests on {} ranges with a concurrency of {}", - ranges.size(), concurrencyFactor); - } - else - { - // our estimate of how many result rows there will be per-range - float resultRowsPerRange = estimateResultRowsPerRange(command, keyspace); - // underestimate how many rows we will get per-range in order to increase the likelihood that we'll - // fetch enough rows in the first round - resultRowsPerRange -= resultRowsPerRange * CONCURRENT_SUBREQUESTS_MARGIN; - concurrencyFactor = resultRowsPerRange == 0.0 - ? 1 - : Math.max(1, Math.min(ranges.size(), (int) Math.ceil(command.limit() / resultRowsPerRange))); + // our estimate of how many result rows there will be per-range + float resultsPerRange = estimateResultsPerRange(command, keyspace); + // underestimate how many rows we will get per-range in order to increase the likelihood that we'll + // fetch enough rows in the first round + resultsPerRange -= resultsPerRange * CONCURRENT_SUBREQUESTS_MARGIN; + int concurrencyFactor = resultsPerRange == 0.0 + ? 1 + : Math.max(1, Math.min(ranges.rangeCount(), (int) Math.ceil(command.limits().count() / resultsPerRange))); + logger.debug("Estimated result rows per range: {}; requested rows: {}, ranges.size(): {}; concurrent range requests: {}", + resultsPerRange, command.limits().count(), ranges.rangeCount(), concurrencyFactor); + Tracing.trace("Submitting range requests on {} ranges with a concurrency of {} ({} rows per range expected)", ranges.rangeCount(), concurrencyFactor, resultsPerRange); - logger.debug("Estimated result rows per range: {}; requested rows: {}, ranges.size(): {}; concurrent range requests: {}", - resultRowsPerRange, - command.limit(), - ranges.size(), - concurrencyFactor); - Tracing.trace("Submitting range requests on {} ranges with a concurrency of {} ({} rows per range expected)", - ranges.size(), - concurrencyFactor, - resultRowsPerRange); - } - - boolean haveSufficientRows = false; - int i = 0; - AbstractBounds nextRange = null; - List nextEndpoints = null; - List nextFilteredEndpoints = null; - while (i < ranges.size()) - { - List>>> scanHandlers = new ArrayList<>(concurrencyFactor); - int concurrentFetchStartingIndex = i; - int concurrentRequests = 0; - while ((i - concurrentFetchStartingIndex) < concurrencyFactor) - { - AbstractBounds range = nextRange == null - ? ranges.get(i) - : nextRange; - List liveEndpoints = nextEndpoints == null - ? getLiveSortedEndpoints(keyspace, range.right) - : nextEndpoints; - List filteredEndpoints = nextFilteredEndpoints == null - ? consistency_level.filterForQuery(keyspace, liveEndpoints) - : nextFilteredEndpoints; - ++i; - ++concurrentRequests; - - // getRestrictedRange has broken the queried range into per-[vnode] token ranges, but this doesn't take - // the replication factor into account. If the intersection of live endpoints for 2 consecutive ranges - // still meets the CL requirements, then we can merge both ranges into the same RangeSliceCommand. - while (i < ranges.size()) - { - nextRange = ranges.get(i); - nextEndpoints = getLiveSortedEndpoints(keyspace, nextRange.right); - nextFilteredEndpoints = consistency_level.filterForQuery(keyspace, nextEndpoints); - - // If the current range right is the min token, we should stop merging because CFS.getRangeSlice - // don't know how to deal with a wrapping range. - // Note: it would be slightly more efficient to have CFS.getRangeSlice on the destination nodes unwraps - // the range if necessary and deal with it. However, we can't start sending wrapped range without breaking - // wire compatibility, so It's likely easier not to bother; - if (range.right.isMinimum()) - break; - - List merged = intersection(liveEndpoints, nextEndpoints); - - // Check if there is enough endpoint for the merge to be possible. - if (!consistency_level.isSufficientLiveNodes(keyspace, merged)) - break; - - List filteredMerged = consistency_level.filterForQuery(keyspace, merged); - - // Estimate whether merging will be a win or not - if (!DatabaseDescriptor.getEndpointSnitch().isWorthMergingForRangeQuery(filteredMerged, filteredEndpoints, nextFilteredEndpoints)) - break; - - // If we get there, merge this range and the next one - range = range.withNewRight(nextRange.right); - liveEndpoints = merged; - filteredEndpoints = filteredMerged; - ++i; - } - - AbstractRangeCommand nodeCmd = command.forSubRange(range); - - // collect replies and resolve according to consistency level - RangeSliceResponseResolver resolver = new RangeSliceResponseResolver(nodeCmd.keyspace, command.timestamp); - List minimalEndpoints = filteredEndpoints.subList(0, Math.min(filteredEndpoints.size(), consistency_level.blockFor(keyspace))); - ReadCallback> handler = new ReadCallback<>(resolver, consistency_level, nodeCmd, minimalEndpoints); - handler.assureSufficientLiveNodes(); - resolver.setSources(filteredEndpoints); - if (filteredEndpoints.size() == 1 - && filteredEndpoints.get(0).equals(FBUtilities.getBroadcastAddress()) - && OPTIMIZE_LOCAL_REQUESTS) - { - StageManager.getStage(Stage.READ).execute(new LocalRangeSliceRunnable(nodeCmd, handler), Tracing.instance.get()); - } - else - { - MessageOut message = nodeCmd.createMessage(); - for (InetAddress endpoint : filteredEndpoints) - { - Tracing.trace("Enqueuing request to {}", endpoint); - MessagingService.instance().sendRRWithFailure(message, endpoint, handler); - } - } - scanHandlers.add(Pair.create(nodeCmd, handler)); - } - Tracing.trace("Submitted {} concurrent range requests covering {} ranges", concurrentRequests, i - concurrentFetchStartingIndex); - - List repairResponses = new ArrayList<>(); - for (Pair>> cmdPairHandler : scanHandlers) - { - ReadCallback> handler = cmdPairHandler.right; - RangeSliceResponseResolver resolver = (RangeSliceResponseResolver)handler.resolver; - - try - { - for (Row row : handler.get()) - { - rows.add(row); - if (countLiveRows) - liveRowCount += row.getLiveCount(command.predicate, command.timestamp); - } - repairResponses.addAll(resolver.repairResults); - } - catch (ReadTimeoutException|ReadFailureException ex) - { - // we timed out or failed waiting for responses - int blockFor = consistency_level.blockFor(keyspace); - int responseCount = resolver.responses.size(); - String gotData = responseCount > 0 - ? resolver.isDataPresent() ? " (including data)" : " (only digests)" - : ""; - - boolean isTimeout = ex instanceof ReadTimeoutException; - if (Tracing.isTracing()) - { - Tracing.trace("{}; received {} of {} responses{} for range {} of {}", - (isTimeout ? "Timed out" : "Failed"), responseCount, blockFor, gotData, i, ranges.size()); - } - else if (logger.isDebugEnabled()) - { - logger.debug("Range slice {}; received {} of {} responses{} for range {} of {}", - (isTimeout ? "timeout" : "failure"), responseCount, blockFor, gotData, i, ranges.size()); - } - throw ex; - } - catch (DigestMismatchException e) - { - throw new AssertionError(e); // no digests in range slices yet - } - - // if we're done, great, otherwise, move to the next range - int count = countLiveRows ? liveRowCount : rows.size(); - if (count >= rowsToBeFetched) - { - haveSufficientRows = true; - break; - } - } - - try - { - FBUtilities.waitOnFutures(repairResponses, DatabaseDescriptor.getWriteRpcTimeout()); - } - catch (TimeoutException ex) - { - // We got all responses, but timed out while repairing - int blockFor = consistency_level.blockFor(keyspace); - if (Tracing.isTracing()) - Tracing.trace("Timed out while read-repairing after receiving all {} data and digest responses", blockFor); - else - logger.debug("Range slice timeout while read-repairing after receiving all {} data and digest responses", blockFor); - throw new ReadTimeoutException(consistency_level, blockFor-1, blockFor, true); - } - - if (haveSufficientRows) - return command.postReconciliationProcessing(rows); - - // we didn't get enough rows in our concurrent fetch; recalculate our concurrency factor - // based on the results we've seen so far (as long as we still have ranges left to query) - if (i < ranges.size()) - { - float fetchedRows = countLiveRows ? liveRowCount : rows.size(); - float remainingRows = rowsToBeFetched - fetchedRows; - float actualRowsPerRange; - if (fetchedRows == 0.0) - { - // we haven't actually gotten any results, so query all remaining ranges at once - actualRowsPerRange = 0.0f; - concurrencyFactor = ranges.size() - i; - } - else - { - actualRowsPerRange = fetchedRows / i; - concurrencyFactor = Math.max(1, Math.min(ranges.size() - i, Math.round(remainingRows / actualRowsPerRange))); - } - logger.debug("Didn't get enough response rows; actual rows per range: {}; remaining rows: {}, new concurrent requests: {}", - actualRowsPerRange, (int) remainingRows, concurrencyFactor); - } - } - } - finally - { - long latency = System.nanoTime() - startTime; - rangeMetrics.addNano(latency); - Keyspace.open(command.keyspace).getColumnFamilyStore(command.columnFamily).metric.coordinatorScanLatency.update(latency, TimeUnit.NANOSECONDS); - } - return command.postReconciliationProcessing(rows); + // Note that in general, a RangeCommandIterator will honor the command limit for each range, but will not enforce it globally. + return command.postReconciliationProcessing(command.limits().filter(new RangeCommandIterator(ranges, command, concurrencyFactor, keyspace, consistencyLevel), command.nowInSec())); } public Map> getSchemaVersions() diff --git a/src/java/org/apache/cassandra/service/StorageService.java b/src/java/org/apache/cassandra/service/StorageService.java index 25ed84937f..93421cee42 100644 --- a/src/java/org/apache/cassandra/service/StorageService.java +++ b/src/java/org/apache/cassandra/service/StorageService.java @@ -67,21 +67,7 @@ import org.apache.cassandra.concurrent.ScheduledExecutors; import org.apache.cassandra.concurrent.Stage; import org.apache.cassandra.concurrent.StageManager; import org.apache.cassandra.config.*; -import org.apache.cassandra.db.BatchlogManager; -import org.apache.cassandra.db.ColumnFamilyStore; -import org.apache.cassandra.db.CounterMutationVerbHandler; -import org.apache.cassandra.db.DecoratedKey; -import org.apache.cassandra.db.DefinitionsUpdateVerbHandler; -import org.apache.cassandra.db.HintedHandOffManager; -import org.apache.cassandra.db.Keyspace; -import org.apache.cassandra.db.MigrationRequestVerbHandler; -import org.apache.cassandra.db.MutationVerbHandler; -import org.apache.cassandra.db.ReadRepairVerbHandler; -import org.apache.cassandra.db.ReadVerbHandler; -import org.apache.cassandra.db.SchemaCheckVerbHandler; -import org.apache.cassandra.db.SnapshotDetailsTabularData; -import org.apache.cassandra.db.SystemKeyspace; -import org.apache.cassandra.db.TruncateVerbHandler; +import org.apache.cassandra.db.*; import org.apache.cassandra.db.commitlog.CommitLog; import org.apache.cassandra.db.compaction.CompactionManager; import org.apache.cassandra.dht.BootStrapper; @@ -311,9 +297,9 @@ public class StorageService extends NotificationBroadcasterSupport implements IE /* register the verb handlers */ MessagingService.instance().registerVerbHandlers(MessagingService.Verb.MUTATION, new MutationVerbHandler()); MessagingService.instance().registerVerbHandlers(MessagingService.Verb.READ_REPAIR, new ReadRepairVerbHandler()); - MessagingService.instance().registerVerbHandlers(MessagingService.Verb.READ, new ReadVerbHandler()); - MessagingService.instance().registerVerbHandlers(MessagingService.Verb.RANGE_SLICE, new RangeSliceVerbHandler()); - MessagingService.instance().registerVerbHandlers(MessagingService.Verb.PAGED_RANGE, new RangeSliceVerbHandler()); + MessagingService.instance().registerVerbHandlers(MessagingService.Verb.READ, new ReadCommandVerbHandler()); + MessagingService.instance().registerVerbHandlers(MessagingService.Verb.RANGE_SLICE, new ReadCommandVerbHandler()); + MessagingService.instance().registerVerbHandlers(MessagingService.Verb.PAGED_RANGE, new ReadCommandVerbHandler()); MessagingService.instance().registerVerbHandlers(MessagingService.Verb.COUNTER_MUTATION, new CounterMutationVerbHandler()); MessagingService.instance().registerVerbHandlers(MessagingService.Verb.TRUNCATE, new TruncateVerbHandler()); MessagingService.instance().registerVerbHandlers(MessagingService.Verb.PAXOS_PREPARE, new PrepareVerbHandler()); @@ -3901,7 +3887,8 @@ public class StorageService extends NotificationBroadcasterSupport implements IE } // Never ever do this at home. Used by tests. - IPartitioner setPartitionerUnsafe(IPartitioner newPartitioner) + @VisibleForTesting + public IPartitioner setPartitionerUnsafe(IPartitioner newPartitioner) { IPartitioner oldPartitioner = DatabaseDescriptor.getPartitioner(); DatabaseDescriptor.setPartitioner(newPartitioner); diff --git a/src/java/org/apache/cassandra/service/pager/AbstractQueryPager.java b/src/java/org/apache/cassandra/service/pager/AbstractQueryPager.java index c330eeaf70..2c16ace858 100644 --- a/src/java/org/apache/cassandra/service/pager/AbstractQueryPager.java +++ b/src/java/org/apache/cassandra/service/pager/AbstractQueryPager.java @@ -17,156 +17,133 @@ */ package org.apache.cassandra.service.pager; -import java.util.*; - -import com.google.common.annotations.VisibleForTesting; - import org.apache.cassandra.config.CFMetaData; -import org.apache.cassandra.config.ColumnDefinition; -import org.apache.cassandra.config.Schema; import org.apache.cassandra.db.*; -import org.apache.cassandra.db.filter.ColumnCounter; -import org.apache.cassandra.db.filter.IDiskAtomFilter; +import org.apache.cassandra.db.rows.*; +import org.apache.cassandra.db.partitions.*; +import org.apache.cassandra.db.filter.DataLimits; import org.apache.cassandra.exceptions.RequestExecutionException; import org.apache.cassandra.exceptions.RequestValidationException; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; +import org.apache.cassandra.service.ClientState; abstract class AbstractQueryPager implements QueryPager { - private static final Logger logger = LoggerFactory.getLogger(AbstractQueryPager.class); - - private final ConsistencyLevel consistencyLevel; - private final boolean localQuery; - - protected final CFMetaData cfm; - protected final IDiskAtomFilter columnFilter; - private final long timestamp; + protected final ReadCommand command; + protected final DataLimits limits; private int remaining; + + // This is the last key we've been reading from (or can still be reading within). This the key for + // which remainingInPartition makes sense: if we're starting another key, we should reset remainingInPartition + // (and this is done in PagerIterator). This can be null (when we start). + private DecoratedKey lastKey; + private int remainingInPartition; + private boolean exhausted; - private boolean shouldFetchExtraRow; - protected AbstractQueryPager(ConsistencyLevel consistencyLevel, - int toFetch, - boolean localQuery, - String keyspace, - String columnFamily, - IDiskAtomFilter columnFilter, - long timestamp) + protected AbstractQueryPager(ReadCommand command) { - this(consistencyLevel, toFetch, localQuery, Schema.instance.getCFMetaData(keyspace, columnFamily), columnFilter, timestamp); + this.command = command; + this.limits = command.limits(); + + this.remaining = limits.count(); + this.remainingInPartition = limits.perPartitionCount(); } - protected AbstractQueryPager(ConsistencyLevel consistencyLevel, - int toFetch, - boolean localQuery, - CFMetaData cfm, - IDiskAtomFilter columnFilter, - long timestamp) + public ReadOrderGroup startOrderGroup() { - this.consistencyLevel = consistencyLevel; - this.localQuery = localQuery; - - this.cfm = cfm; - this.columnFilter = columnFilter; - this.timestamp = timestamp; - - this.remaining = toFetch; + return command.startOrderGroup(); } - - public List fetchPage(int pageSize) throws RequestValidationException, RequestExecutionException + public PartitionIterator fetchPage(int pageSize, ConsistencyLevel consistency, ClientState clientState) throws RequestValidationException, RequestExecutionException { if (isExhausted()) - return Collections.emptyList(); + return PartitionIterators.EMPTY; - int currentPageSize = nextPageSize(pageSize); - List rows = filterEmpty(queryNextPage(currentPageSize, consistencyLevel, localQuery)); - - if (rows.isEmpty()) - { - logger.debug("Got empty set of rows, considering pager exhausted"); - exhausted = true; - return Collections.emptyList(); - } - - int liveCount = getPageLiveCount(rows); - logger.debug("Fetched {} live rows", liveCount); - - // Because SP.getRangeSlice doesn't trim the result (see SP.trim()), liveCount may be greater than what asked - // (currentPageSize). This would throw off the paging logic so we trim the excess. It's not extremely efficient - // but most of the time there should be nothing or very little to trim. - if (liveCount > currentPageSize) - { - rows = discardLast(rows, liveCount - currentPageSize); - liveCount = currentPageSize; - } - - remaining -= liveCount; - - // If we've got less than requested, there is no more query to do (but - // we still need to return the current page) - if (liveCount < currentPageSize) - { - logger.debug("Got result ({}) smaller than page size ({}), considering pager exhausted", liveCount, currentPageSize); - exhausted = true; - } - - // If it's not the first query and the first column is the last one returned (likely - // but not certain since paging can race with deletes/expiration), then remove the - // first column. - if (containsPreviousLast(rows.get(0))) - { - rows = discardFirst(rows); - remaining++; - } - // Otherwise, if 'shouldFetchExtraRow' was set, we queried for one more than the page size, - // so if the page is full, trim the last entry - else if (shouldFetchExtraRow && !exhausted) - { - // We've asked for one more than necessary - rows = discardLast(rows); - remaining++; - } - - logger.debug("Remaining rows to page: {}", remaining); - - if (!isExhausted()) - shouldFetchExtraRow = recordLast(rows.get(rows.size() - 1)); - - return rows; + pageSize = Math.min(pageSize, remaining); + return new PagerIterator(nextPageReadCommand(pageSize).execute(consistency, clientState), limits.forPaging(pageSize), command.nowInSec()); } - private List filterEmpty(List result) + public PartitionIterator fetchPageInternal(int pageSize, ReadOrderGroup orderGroup) throws RequestValidationException, RequestExecutionException { - for (Row row : result) - { - if (row.cf == null || !row.cf.hasColumns()) - { - List newResult = new ArrayList(result.size() - 1); - for (Row row2 : result) - { - if (row2.cf == null || !row2.cf.hasColumns()) - continue; + if (isExhausted()) + return PartitionIterators.EMPTY; - newResult.add(row2); - } - return newResult; + pageSize = Math.min(pageSize, remaining); + return new PagerIterator(nextPageReadCommand(pageSize).executeInternal(orderGroup), limits.forPaging(pageSize), command.nowInSec()); + } + + private class PagerIterator extends CountingPartitionIterator + { + private final DataLimits pageLimits; + + private Row lastRow; + + private PagerIterator(PartitionIterator iter, DataLimits pageLimits, int nowInSec) + { + super(iter, pageLimits, nowInSec); + this.pageLimits = pageLimits; + } + + @Override + @SuppressWarnings("resource") // iter is closed by closing the result + public RowIterator next() + { + RowIterator iter = super.next(); + try + { + DecoratedKey key = iter.partitionKey(); + if (lastKey == null || !lastKey.equals(key)) + remainingInPartition = limits.perPartitionCount(); + + lastKey = key; + return new RowPagerIterator(iter); + } + catch (RuntimeException e) + { + iter.close(); + throw e; + } + } + + @Override + public void close() + { + super.close(); + recordLast(lastKey, lastRow); + + int counted = counter.counted(); + remaining -= counted; + remainingInPartition -= counter.countedInCurrentPartition(); + exhausted = counted < pageLimits.count(); + } + + private class RowPagerIterator extends WrappingRowIterator + { + RowPagerIterator(RowIterator iter) + { + super(iter); + } + + @Override + public Row next() + { + lastRow = super.next(); + return lastRow; } } - return result; } - protected void restoreState(int remaining, boolean shouldFetchExtraRow) + protected void restoreState(DecoratedKey lastKey, int remaining, int remainingInPartition) { + this.lastKey = lastKey; this.remaining = remaining; - this.shouldFetchExtraRow = shouldFetchExtraRow; + this.remainingInPartition = remainingInPartition; } public boolean isExhausted() { - return exhausted || remaining == 0; + return exhausted || remaining == 0 || ((this instanceof SinglePartitionPager) && remainingInPartition == 0); } public int maxRemaining() @@ -174,220 +151,11 @@ abstract class AbstractQueryPager implements QueryPager return remaining; } - public long timestamp() + protected int remainingInPartition() { - return timestamp; + return remainingInPartition; } - private int nextPageSize(int pageSize) - { - return Math.min(remaining, pageSize) + (shouldFetchExtraRow ? 1 : 0); - } - - public ColumnCounter columnCounter() - { - return columnFilter.columnCounter(cfm.comparator, timestamp); - } - - protected abstract List queryNextPage(int pageSize, ConsistencyLevel consistency, boolean localQuery) throws RequestValidationException, RequestExecutionException; - - /** - * Checks to see if the first row of a new page contains the last row from the previous page. - * @param first the first row of the new page - * @return true if first contains the last from from the previous page and it is live, false otherwise - */ - protected abstract boolean containsPreviousLast(Row first); - - /** - * Saves the paging state by recording the last seen partition key and cell name (where applicable). - * @param last the last row in the current page - * @return true if an extra row should be fetched in the next page,false otherwise - */ - protected abstract boolean recordLast(Row last); - - protected abstract boolean isReversed(); - - private List discardFirst(List rows) - { - return discardFirst(rows, 1); - } - - @VisibleForTesting - List discardFirst(List rows, int toDiscard) - { - if (toDiscard == 0 || rows.isEmpty()) - return rows; - - int i = 0; - DecoratedKey firstKey = null; - ColumnFamily firstCf = null; - while (toDiscard > 0 && i < rows.size()) - { - Row first = rows.get(i++); - firstKey = first.key; - firstCf = first.cf.cloneMeShallow(isReversed()); - toDiscard -= isReversed() - ? discardLast(first.cf, toDiscard, firstCf) - : discardFirst(first.cf, toDiscard, firstCf); - } - - // If there is less live data than to discard, all is discarded - if (toDiscard > 0) - return Collections.emptyList(); - - // i is the index of the first row that we are sure to keep. On top of that, - // we also keep firstCf is it hasn't been fully emptied by the last iteration above. - int count = firstCf.getColumnCount(); - int newSize = rows.size() - (count == 0 ? i : i - 1); - List newRows = new ArrayList(newSize); - if (count != 0) - newRows.add(new Row(firstKey, firstCf)); - newRows.addAll(rows.subList(i, rows.size())); - - return newRows; - } - - private List discardLast(List rows) - { - return discardLast(rows, 1); - } - - @VisibleForTesting - List discardLast(List rows, int toDiscard) - { - if (toDiscard == 0 || rows.isEmpty()) - return rows; - - int i = rows.size()-1; - DecoratedKey lastKey = null; - ColumnFamily lastCf = null; - while (toDiscard > 0 && i >= 0) - { - Row last = rows.get(i--); - lastKey = last.key; - lastCf = last.cf.cloneMeShallow(isReversed()); - toDiscard -= isReversed() - ? discardFirst(last.cf, toDiscard, lastCf) - : discardLast(last.cf, toDiscard, lastCf); - } - - // If there is less live data than to discard, all is discarded - if (toDiscard > 0) - return Collections.emptyList(); - - // i is the index of the last row that we are sure to keep. On top of that, - // we also keep lastCf is it hasn't been fully emptied by the last iteration above. - int count = lastCf.getColumnCount(); - int newSize = count == 0 ? i+1 : i+2; - List newRows = new ArrayList(newSize); - newRows.addAll(rows.subList(0, i+1)); - if (count != 0) - newRows.add(new Row(lastKey, lastCf)); - - return newRows; - } - - private int getPageLiveCount(List page) - { - int count = 0; - for (Row row : page) - count += columnCounter().countAll(row.cf).live(); - return count; - } - - private int discardFirst(ColumnFamily cf, int toDiscard, ColumnFamily newCf) - { - boolean isReversed = isReversed(); - DeletionInfo.InOrderTester tester = cf.deletionInfo().inOrderTester(isReversed); - return isReversed - ? discardTail(cf, toDiscard, newCf, cf.reverseIterator(), tester) - : discardHead(toDiscard, newCf, cf.iterator(), tester); - } - - private int discardLast(ColumnFamily cf, int toDiscard, ColumnFamily newCf) - { - boolean isReversed = isReversed(); - DeletionInfo.InOrderTester tester = cf.deletionInfo().inOrderTester(isReversed); - return isReversed - ? discardHead(toDiscard, newCf, cf.reverseIterator(), tester) - : discardTail(cf, toDiscard, newCf, cf.iterator(), tester); - } - - private int discardHead(int toDiscard, ColumnFamily copy, Iterator iter, DeletionInfo.InOrderTester tester) - { - ColumnCounter counter = columnCounter(); - - List staticCells = new ArrayList<>(cfm.staticColumns().size()); - - // Discard the first 'toDiscard' live, non-static cells - while (iter.hasNext()) - { - Cell c = iter.next(); - - // if it's a static column, don't count it and save it to add to the trimmed results - ColumnDefinition columnDef = cfm.getColumnDefinition(c.name()); - if (columnDef != null && columnDef.kind == ColumnDefinition.Kind.STATIC) - { - staticCells.add(c); - continue; - } - - counter.count(c, tester); - - // once we've discarded the required amount, add the rest - if (counter.live() > toDiscard) - { - for (Cell staticCell : staticCells) - copy.addColumn(staticCell); - - copy.addColumn(c); - while (iter.hasNext()) - copy.addColumn(iter.next()); - } - } - return Math.min(counter.live(), toDiscard); - } - - private int discardTail(ColumnFamily cf, int toDiscard, ColumnFamily copy, Iterator iter, DeletionInfo.InOrderTester tester) - { - // Redoing the counting like that is not extremely efficient. - // This is called only for reversed slices or in the case of a race between - // paging and a deletion (pretty unlikely), so this is probably acceptable. - int liveCount = columnCounter().countAll(cf).live(); - - ColumnCounter counter = columnCounter(); - // Discard the last 'toDiscard' live (so stop adding as sound as we're past 'liveCount - toDiscard') - while (iter.hasNext()) - { - Cell c = iter.next(); - counter.count(c, tester); - if (counter.live() > liveCount - toDiscard) - break; - - copy.addColumn(c); - } - return Math.min(liveCount, toDiscard); - } - - /** - * Returns the first non-static cell in the ColumnFamily. This is necessary to avoid recording a static column - * as the "last" cell seen in a reversed query. Because we will always query static columns alongside the normal - * data for a page, they are not a good indicator of where paging should resume. When we begin the next page, we - * need to start from the last non-static cell. - */ - protected Cell firstNonStaticCell(ColumnFamily cf) - { - for (Cell cell : cf) - { - ColumnDefinition def = cfm.getColumnDefinition(cell.name()); - if (def == null || def.kind != ColumnDefinition.Kind.STATIC) - return cell; - } - return null; - } - - protected static Cell lastCell(ColumnFamily cf) - { - return cf.getReverseSortedColumns().iterator().next(); - } + protected abstract ReadCommand nextPageReadCommand(int pageSize); + protected abstract void recordLast(DecoratedKey key, Row row); } diff --git a/src/java/org/apache/cassandra/service/pager/MultiPartitionPager.java b/src/java/org/apache/cassandra/service/pager/MultiPartitionPager.java index 35d0971c10..4fb14290c3 100644 --- a/src/java/org/apache/cassandra/service/pager/MultiPartitionPager.java +++ b/src/java/org/apache/cassandra/service/pager/MultiPartitionPager.java @@ -17,10 +17,14 @@ */ package org.apache.cassandra.service.pager; -import java.util.ArrayList; import java.util.List; +import com.google.common.collect.AbstractIterator; + import org.apache.cassandra.db.*; +import org.apache.cassandra.db.rows.*; +import org.apache.cassandra.db.filter.DataLimits; +import org.apache.cassandra.db.partitions.*; import org.apache.cassandra.exceptions.RequestValidationException; import org.apache.cassandra.exceptions.RequestExecutionException; import org.apache.cassandra.service.ClientState; @@ -39,53 +43,44 @@ import org.apache.cassandra.service.ClientState; * cfs meanRowSize to decide if parallelizing some of the command might be worth it while being confident we don't * blow out memory. */ -class MultiPartitionPager implements QueryPager +public class MultiPartitionPager implements QueryPager { private final SinglePartitionPager[] pagers; - private final long timestamp; + private final DataLimits limit; + + private final int nowInSec; private int remaining; private int current; - MultiPartitionPager(List commands, ConsistencyLevel consistencyLevel, ClientState cState, boolean localQuery, PagingState state, int limitForQuery) + public MultiPartitionPager(SinglePartitionReadCommand.Group group, PagingState state) { + this.limit = group.limits(); + this.nowInSec = group.nowInSec(); + int i = 0; // If it's not the beginning (state != null), we need to find where we were and skip previous commands // since they are done. if (state != null) - for (; i < commands.size(); i++) - if (commands.get(i).key.equals(state.partitionKey)) + for (; i < group.commands.size(); i++) + if (group.commands.get(i).partitionKey().getKey().equals(state.partitionKey)) break; - if (i >= commands.size()) + if (i >= group.commands.size()) { pagers = null; - timestamp = -1; return; } - pagers = new SinglePartitionPager[commands.size() - i]; + pagers = new SinglePartitionPager[group.commands.size() - i]; // 'i' is on the first non exhausted pager for the previous page (or the first one) - pagers[0] = makePager(commands.get(i), consistencyLevel, cState, localQuery, state); - timestamp = commands.get(i).timestamp; + pagers[0] = group.commands.get(i).getPager(state); // Following ones haven't been started yet - for (int j = i + 1; j < commands.size(); j++) - { - ReadCommand command = commands.get(j); - if (command.timestamp != timestamp) - throw new IllegalArgumentException("All commands must have the same timestamp or weird results may happen."); - pagers[j - i] = makePager(command, consistencyLevel, cState, localQuery, null); - } + for (int j = i + 1; j < group.commands.size(); j++) + pagers[j - i] = group.commands.get(j).getPager(null); - remaining = state == null ? limitForQuery : state.remaining; - } - - private static SinglePartitionPager makePager(ReadCommand command, ConsistencyLevel consistencyLevel, ClientState cState, boolean localQuery, PagingState state) - { - return command instanceof SliceFromReadCommand - ? new SliceQueryPager((SliceFromReadCommand)command, consistencyLevel, cState, localQuery, state) - : new NamesQueryPager((SliceByNamesReadCommand)command, consistencyLevel, cState, localQuery); + remaining = state == null ? limit.count() : state.remaining; } public PagingState state() @@ -95,7 +90,7 @@ class MultiPartitionPager implements QueryPager return null; PagingState state = pagers[current].state(); - return new PagingState(pagers[current].key(), state == null ? null : state.cellName, remaining); + return new PagingState(pagers[current].key(), state == null ? null : state.cellName, remaining, Integer.MAX_VALUE); } public boolean isExhausted() @@ -113,35 +108,92 @@ class MultiPartitionPager implements QueryPager return true; } - public List fetchPage(int pageSize) throws RequestValidationException, RequestExecutionException + public ReadOrderGroup startOrderGroup() { - List result = new ArrayList(); - - int remainingThisQuery = Math.min(remaining, pageSize); - while (remainingThisQuery > 0 && !isExhausted()) + // Note that for all pagers, the only difference is the partition key to which it applies, so in practice we + // can use any of the sub-pager ReadOrderGroup group to protect the whole pager + for (int i = current; i < pagers.length; i++) { - // isExhausted has set us on the first non-exhausted pager - List page = pagers[current].fetchPage(remainingThisQuery); - if (page.isEmpty()) - continue; + if (pagers[i] != null) + return pagers[i].startOrderGroup(); + } + throw new AssertionError("Shouldn't be called on an exhausted pager"); + } - Row row = page.get(0); - int fetched = pagers[current].columnCounter().countAll(row.cf).live(); - remaining -= fetched; - remainingThisQuery -= fetched; - result.add(row); + @SuppressWarnings("resource") + public PartitionIterator fetchPage(int pageSize, ConsistencyLevel consistency, ClientState clientState) throws RequestValidationException, RequestExecutionException + { + int toQuery = Math.min(remaining, pageSize); + PagersIterator iter = new PagersIterator(toQuery, consistency, clientState, null); + CountingPartitionIterator countingIter = new CountingPartitionIterator(iter, limit.forPaging(toQuery), nowInSec); + iter.setCounter(countingIter.counter()); + return countingIter; + } + + public PartitionIterator fetchPageInternal(int pageSize, ReadOrderGroup orderGroup) throws RequestValidationException, RequestExecutionException + { + int toQuery = Math.min(remaining, pageSize); + PagersIterator iter = new PagersIterator(toQuery, null, null, orderGroup); + CountingPartitionIterator countingIter = new CountingPartitionIterator(iter, limit.forPaging(toQuery), nowInSec); + iter.setCounter(countingIter.counter()); + return countingIter; + } + + private class PagersIterator extends AbstractIterator implements PartitionIterator + { + private final int pageSize; + private PartitionIterator result; + private DataLimits.Counter counter; + + // For "normal" queries + private final ConsistencyLevel consistency; + private final ClientState clientState; + + // For internal queries + private final ReadOrderGroup orderGroup; + + public PagersIterator(int pageSize, ConsistencyLevel consistency, ClientState clientState, ReadOrderGroup orderGroup) + { + this.pageSize = pageSize; + this.consistency = consistency; + this.clientState = clientState; + this.orderGroup = orderGroup; } - return result; + public void setCounter(DataLimits.Counter counter) + { + this.counter = counter; + } + + protected RowIterator computeNext() + { + while (result == null || !result.hasNext()) + { + // This sets us on the first non-exhausted pager + if (isExhausted()) + return endOfData(); + + if (result != null) + result.close(); + + int toQuery = pageSize - counter.counted(); + result = consistency == null + ? pagers[current].fetchPageInternal(toQuery, orderGroup) + : pagers[current].fetchPage(toQuery, consistency, clientState); + } + return result.next(); + } + + public void close() + { + remaining -= counter.counted(); + if (result != null) + result.close(); + } } public int maxRemaining() { return remaining; } - - public long timestamp() - { - return timestamp; - } } diff --git a/src/java/org/apache/cassandra/service/pager/NamesQueryPager.java b/src/java/org/apache/cassandra/service/pager/NamesQueryPager.java deleted file mode 100644 index d03e58265f..0000000000 --- a/src/java/org/apache/cassandra/service/pager/NamesQueryPager.java +++ /dev/null @@ -1,108 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.apache.cassandra.service.pager; - -import java.nio.ByteBuffer; -import java.util.Collections; -import java.util.List; - -import org.apache.cassandra.db.*; -import org.apache.cassandra.db.filter.ColumnCounter; -import org.apache.cassandra.exceptions.RequestValidationException; -import org.apache.cassandra.exceptions.RequestExecutionException; -import org.apache.cassandra.service.ClientState; -import org.apache.cassandra.service.StorageProxy; - -/** - * Pager over a SliceByNamesReadCommand. - */ -public class NamesQueryPager implements SinglePartitionPager -{ - private final SliceByNamesReadCommand command; - private final ConsistencyLevel consistencyLevel; - private final ClientState state; - private final boolean localQuery; - - private volatile boolean queried; - - /** - * For now, we'll only use this in CQL3. In there, as name query can never - * yield more than one CQL3 row, there is no need for paging and so this is straight-forward. - * - * For thrift, we could imagine needing to page, though even then it's very - * unlikely unless the pageSize is very small. - * - * In any case we currently assert in fetchPage if it's a "thrift" query (i.e. a query that - * count every cell individually) and the names filter asks for more than pageSize columns. - */ - // Don't use directly, use QueryPagers method instead - NamesQueryPager(SliceByNamesReadCommand command, ConsistencyLevel consistencyLevel, ClientState state, boolean localQuery) - { - this.command = command; - this.consistencyLevel = consistencyLevel; - this.state = state; - this.localQuery = localQuery; - } - - public ByteBuffer key() - { - return command.key; - } - - public ColumnCounter columnCounter() - { - // We know NamesQueryFilter.columnCounter don't care about his argument - return command.filter.columnCounter(null, command.timestamp); - } - - public PagingState state() - { - return null; - } - - public boolean isExhausted() - { - return queried; - } - - public List fetchPage(int pageSize) throws RequestValidationException, RequestExecutionException - { - assert command.filter.countCQL3Rows() || command.filter.columns.size() <= pageSize; - - if (isExhausted()) - return Collections.emptyList(); - - queried = true; - return localQuery - ? Collections.singletonList(command.getRow(Keyspace.open(command.ksName))) - : StorageProxy.read(Collections.singletonList(command), consistencyLevel, state); - } - - public int maxRemaining() - { - if (queried) - return 0; - - return command.filter.countCQL3Rows() ? 1 : command.filter.columns.size(); - } - - public long timestamp() - { - return command.timestamp; - } -} diff --git a/src/java/org/apache/cassandra/service/pager/PagingState.java b/src/java/org/apache/cassandra/service/pager/PagingState.java index f1688800f9..685dc3fa1c 100644 --- a/src/java/org/apache/cassandra/service/pager/PagingState.java +++ b/src/java/org/apache/cassandra/service/pager/PagingState.java @@ -31,12 +31,14 @@ public class PagingState public final ByteBuffer partitionKey; public final ByteBuffer cellName; public final int remaining; + public final int remainingInPartition; - public PagingState(ByteBuffer partitionKey, ByteBuffer cellName, int remaining) + public PagingState(ByteBuffer partitionKey, ByteBuffer cellName, int remaining, int remainingInPartition) { this.partitionKey = partitionKey == null ? ByteBufferUtil.EMPTY_BYTE_BUFFER : partitionKey; this.cellName = cellName == null ? ByteBufferUtil.EMPTY_BYTE_BUFFER : cellName; this.remaining = remaining; + this.remainingInPartition = remainingInPartition; } public static PagingState deserialize(ByteBuffer bytes) @@ -50,7 +52,12 @@ public class PagingState ByteBuffer pk = ByteBufferUtil.readWithShortLength(in); ByteBuffer cn = ByteBufferUtil.readWithShortLength(in); int remaining = in.readInt(); - return new PagingState(pk, cn, remaining); + // Note that while 'in.available()' is theoretically an estimate of how many bytes are available + // without blocking, we know that since we're reading a ByteBuffer it will be exactly how many + // bytes remain to be read. And the reason we want to condition this is for backward compatility + // as we used to not set this. + int remainingInPartition = in.available() > 0 ? in.readInt() : Integer.MAX_VALUE; + return new PagingState(pk, cn, remaining, remainingInPartition); } catch (IOException e) { @@ -65,6 +72,7 @@ public class PagingState ByteBufferUtil.writeWithShortLength(partitionKey, out); ByteBufferUtil.writeWithShortLength(cellName, out); out.writeInt(remaining); + out.writeInt(remainingInPartition); return out.buffer(); } catch (IOException e) @@ -77,12 +85,16 @@ public class PagingState { return 2 + partitionKey.remaining() + 2 + cellName.remaining() - + 4; + + 8; // remaining & remainingInPartition } @Override public String toString() { - return String.format("PagingState(key=%s, cellname=%s, remaining=%d", ByteBufferUtil.bytesToHex(partitionKey), ByteBufferUtil.bytesToHex(cellName), remaining); + return String.format("PagingState(key=%s, cellname=%s, remaining=%d, remainingInPartition=%d", + ByteBufferUtil.bytesToHex(partitionKey), + ByteBufferUtil.bytesToHex(cellName), + remaining, + remainingInPartition); } } diff --git a/src/java/org/apache/cassandra/service/pager/QueryPager.java b/src/java/org/apache/cassandra/service/pager/QueryPager.java index ab2dad741c..a69335df16 100644 --- a/src/java/org/apache/cassandra/service/pager/QueryPager.java +++ b/src/java/org/apache/cassandra/service/pager/QueryPager.java @@ -17,11 +17,13 @@ */ package org.apache.cassandra.service.pager; -import java.util.List; - -import org.apache.cassandra.db.Row; +import org.apache.cassandra.db.ConsistencyLevel; +import org.apache.cassandra.db.ReadOrderGroup; +import org.apache.cassandra.db.partitions.PartitionIterator; +import org.apache.cassandra.db.partitions.PartitionIterators; import org.apache.cassandra.exceptions.RequestExecutionException; import org.apache.cassandra.exceptions.RequestValidationException; +import org.apache.cassandra.service.ClientState; /** * Perform a query, paging it by page of a given size. @@ -44,13 +46,69 @@ import org.apache.cassandra.exceptions.RequestValidationException; */ public interface QueryPager { + public static final QueryPager EMPTY = new QueryPager() + { + public ReadOrderGroup startOrderGroup() + { + return ReadOrderGroup.emptyGroup(); + } + + public PartitionIterator fetchPage(int pageSize, ConsistencyLevel consistency, ClientState clientState) throws RequestValidationException, RequestExecutionException + { + return PartitionIterators.EMPTY; + } + + public PartitionIterator fetchPageInternal(int pageSize, ReadOrderGroup orderGroup) throws RequestValidationException, RequestExecutionException + { + return PartitionIterators.EMPTY; + } + + public boolean isExhausted() + { + return true; + } + + public int maxRemaining() + { + return 0; + } + + public PagingState state() + { + return null; + } + }; + /** * Fetches the next page. * * @param pageSize the maximum number of elements to return in the next page. + * @param consistency the consistency level to achieve for the query. + * @param clientState the {@code ClientState} for the query. In practice, this can be null unless + * {@code consistency} is a serial consistency. * @return the page of result. */ - public List fetchPage(int pageSize) throws RequestValidationException, RequestExecutionException; + public PartitionIterator fetchPage(int pageSize, ConsistencyLevel consistency, ClientState clientState) throws RequestValidationException, RequestExecutionException; + + /** + * Starts a new read operation. + *

+ * This must be called before {@link fetchPageInternal} and passed to it to protect the read. + * The returned object must be closed on all path and it is thus strongly advised to + * use it in a try-with-ressource construction. + * + * @return a newly started order group for this {@code QueryPager}. + */ + public ReadOrderGroup startOrderGroup(); + + /** + * Fetches the next page internally (in other, this does a local query). + * + * @param pageSize the maximum number of elements to return in the next page. + * @param orderGroup the {@code ReadOrderGroup} protecting the read. + * @return the page of result. + */ + public PartitionIterator fetchPageInternal(int pageSize, ReadOrderGroup orderGroup) throws RequestValidationException, RequestExecutionException; /** * Whether or not this pager is exhausted, i.e. whether or not a call to diff --git a/src/java/org/apache/cassandra/service/pager/QueryPagers.java b/src/java/org/apache/cassandra/service/pager/QueryPagers.java index f933ccb026..618ca32c41 100644 --- a/src/java/org/apache/cassandra/service/pager/QueryPagers.java +++ b/src/java/org/apache/cassandra/service/pager/QueryPagers.java @@ -17,180 +17,47 @@ */ package org.apache.cassandra.service.pager; -import java.nio.ByteBuffer; -import java.util.Iterator; -import java.util.List; - -import org.apache.cassandra.config.Schema; +import org.apache.cassandra.config.CFMetaData; import org.apache.cassandra.db.*; -import org.apache.cassandra.db.filter.ColumnCounter; -import org.apache.cassandra.db.filter.NamesQueryFilter; -import org.apache.cassandra.db.filter.SliceQueryFilter; -import org.apache.cassandra.db.columniterator.IdentityQueryFilter; +import org.apache.cassandra.db.filter.*; +import org.apache.cassandra.db.partitions.*; import org.apache.cassandra.exceptions.RequestExecutionException; import org.apache.cassandra.exceptions.RequestValidationException; import org.apache.cassandra.service.ClientState; /** - * Static utility methods to create query pagers. + * Static utility methods for paging. */ public class QueryPagers { private QueryPagers() {}; - private static int maxQueried(ReadCommand command) - { - if (command instanceof SliceByNamesReadCommand) - { - NamesQueryFilter filter = ((SliceByNamesReadCommand)command).filter; - return filter.countCQL3Rows() ? 1 : filter.columns.size(); - } - else - { - SliceQueryFilter filter = ((SliceFromReadCommand)command).filter; - return filter.count; - } - } - - public static boolean mayNeedPaging(Pageable command, int pageSize) - { - if (command instanceof Pageable.ReadCommands) - { - List commands = ((Pageable.ReadCommands)command).commands; - - // Using long on purpose, as we could overflow otherwise - long maxQueried = 0; - for (ReadCommand readCmd : commands) - maxQueried += maxQueried(readCmd); - - return maxQueried > pageSize; - } - else if (command instanceof ReadCommand) - { - return maxQueried((ReadCommand)command) > pageSize; - } - else - { - assert command instanceof RangeSliceCommand; - RangeSliceCommand rsc = (RangeSliceCommand)command; - // We don't support paging for thrift in general because the way thrift RangeSliceCommand count rows - // independently of cells makes things harder (see RangeSliceQueryPager). The one case where we do - // get a RangeSliceCommand from CQL3 without the countCQL3Rows flag set is for DISTINCT. In that case - // however, the underlying sliceQueryFilter count is 1, so that the RSC limit is still a limit on the - // number of CQL3 rows returned. - assert rsc.countCQL3Rows || (rsc.predicate instanceof SliceQueryFilter && ((SliceQueryFilter)rsc.predicate).count == 1); - return rsc.maxResults > pageSize; - } - } - - private static QueryPager pager(ReadCommand command, ConsistencyLevel consistencyLevel, ClientState cState, boolean local, PagingState state) - { - if (command instanceof SliceByNamesReadCommand) - return new NamesQueryPager((SliceByNamesReadCommand)command, consistencyLevel, cState, local); - else - return new SliceQueryPager((SliceFromReadCommand)command, consistencyLevel, cState, local, state); - } - - private static QueryPager pager(Pageable command, ConsistencyLevel consistencyLevel, ClientState cState, boolean local, PagingState state) - { - if (command instanceof Pageable.ReadCommands) - { - List commands = ((Pageable.ReadCommands)command).commands; - if (commands.size() == 1) - return pager(commands.get(0), consistencyLevel, cState, local, state); - - return new MultiPartitionPager(commands, consistencyLevel, cState, local, state, ((Pageable.ReadCommands) command).limitForQuery); - } - else if (command instanceof ReadCommand) - { - return pager((ReadCommand)command, consistencyLevel, cState, local, state); - } - else - { - assert command instanceof RangeSliceCommand; - RangeSliceCommand rangeCommand = (RangeSliceCommand)command; - if (rangeCommand.predicate instanceof NamesQueryFilter) - return new RangeNamesQueryPager(rangeCommand, consistencyLevel, local, state); - else - return new RangeSliceQueryPager(rangeCommand, consistencyLevel, local, state); - } - } - - public static QueryPager pager(Pageable command, ConsistencyLevel consistencyLevel, ClientState cState) - { - return pager(command, consistencyLevel, cState, false, null); - } - - public static QueryPager pager(Pageable command, ConsistencyLevel consistencyLevel, ClientState cState, PagingState state) - { - return pager(command, consistencyLevel, cState, false, state); - } - - public static QueryPager localPager(Pageable command) - { - return pager(command, null, null, true, null); - } - - /** - * Convenience method to (locally) page an internal row. - * Used to 2ndary index a wide row without dying. - */ - public static Iterator pageRowLocally(final ColumnFamilyStore cfs, ByteBuffer key, final int pageSize) - { - SliceFromReadCommand command = new SliceFromReadCommand(cfs.metadata.ksName, key, cfs.name, System.currentTimeMillis(), new IdentityQueryFilter()); - final SliceQueryPager pager = new SliceQueryPager(command, null, null, true); - - return new Iterator() - { - // We don't use AbstractIterator because we don't want hasNext() to do an actual query - public boolean hasNext() - { - return !pager.isExhausted(); - } - - public ColumnFamily next() - { - try - { - List rows = pager.fetchPage(pageSize); - ColumnFamily cf = rows.isEmpty() ? null : rows.get(0).cf; - return cf == null ? ArrayBackedSortedColumns.factory.create(cfs.metadata) : cf; - } - catch (Exception e) - { - throw new RuntimeException(e); - } - } - - public void remove() - { - throw new UnsupportedOperationException(); - } - }; - } - /** * Convenience method that count (live) cells/rows for a given slice of a row, but page underneath. */ - public static int countPaged(String keyspace, - String columnFamily, - ByteBuffer key, - SliceQueryFilter filter, + public static int countPaged(CFMetaData metadata, + DecoratedKey key, + ColumnFilter columnFilter, + ClusteringIndexFilter filter, + DataLimits limits, ConsistencyLevel consistencyLevel, - ClientState cState, + ClientState state, final int pageSize, - long now) throws RequestValidationException, RequestExecutionException + int nowInSec, + boolean isForThrift) throws RequestValidationException, RequestExecutionException { - SliceFromReadCommand command = new SliceFromReadCommand(keyspace, key, columnFamily, now, filter); - final SliceQueryPager pager = new SliceQueryPager(command, consistencyLevel, cState, false); + SinglePartitionReadCommand command = SinglePartitionReadCommand.create(isForThrift, metadata, nowInSec, columnFilter, RowFilter.NONE, limits, key, filter); + final SinglePartitionPager pager = new SinglePartitionPager(command, null); - ColumnCounter counter = filter.columnCounter(Schema.instance.getCFMetaData(keyspace, columnFamily).comparator, now); + int count = 0; while (!pager.isExhausted()) { - List next = pager.fetchPage(pageSize); - if (!next.isEmpty()) - counter.countAll(next.get(0).cf); + try (CountingPartitionIterator iter = new CountingPartitionIterator(pager.fetchPage(pageSize, consistencyLevel, state), limits, nowInSec)) + { + PartitionIterators.consume(iter); + count += iter.counter().counted(); + } } - return counter.live(); + return count; } } diff --git a/src/java/org/apache/cassandra/service/pager/RangeNamesQueryPager.java b/src/java/org/apache/cassandra/service/pager/RangeNamesQueryPager.java index 50d128006f..fffb4e1381 100644 --- a/src/java/org/apache/cassandra/service/pager/RangeNamesQueryPager.java +++ b/src/java/org/apache/cassandra/service/pager/RangeNamesQueryPager.java @@ -17,13 +17,11 @@ */ package org.apache.cassandra.service.pager; -import java.util.List; - import org.apache.cassandra.db.*; -import org.apache.cassandra.db.filter.NamesQueryFilter; +import org.apache.cassandra.db.rows.*; +import org.apache.cassandra.db.partitions.*; import org.apache.cassandra.dht.*; import org.apache.cassandra.exceptions.RequestExecutionException; -import org.apache.cassandra.service.StorageProxy; import org.apache.cassandra.service.StorageService; /** @@ -37,25 +35,17 @@ import org.apache.cassandra.service.StorageService; */ public class RangeNamesQueryPager extends AbstractQueryPager { - private final RangeSliceCommand command; private volatile DecoratedKey lastReturnedKey; - // Don't use directly, use QueryPagers method instead - RangeNamesQueryPager(RangeSliceCommand command, ConsistencyLevel consistencyLevel, boolean localQuery) + public RangeNamesQueryPager(PartitionRangeReadCommand command, PagingState state) { - super(consistencyLevel, command.maxResults, localQuery, command.keyspace, command.columnFamily, command.predicate, command.timestamp); - this.command = command; - assert columnFilter instanceof NamesQueryFilter && ((NamesQueryFilter)columnFilter).countCQL3Rows(); - } - - RangeNamesQueryPager(RangeSliceCommand command, ConsistencyLevel consistencyLevel, boolean localQuery, PagingState state) - { - this(command, consistencyLevel, localQuery); + super(command); + assert command.isNamesQuery(); if (state != null) { lastReturnedKey = StorageService.getPartitioner().decorateKey(state.partitionKey); - restoreState(state.remaining, true); + restoreState(lastReturnedKey, state.remaining, state.remainingInPartition); } } @@ -63,51 +53,36 @@ public class RangeNamesQueryPager extends AbstractQueryPager { return lastReturnedKey == null ? null - : new PagingState(lastReturnedKey.getKey(), null, maxRemaining()); + : new PagingState(lastReturnedKey.getKey(), null, maxRemaining(), remainingInPartition()); } - protected List queryNextPage(int pageSize, ConsistencyLevel consistencyLevel, boolean localQuery) + protected ReadCommand nextPageReadCommand(int pageSize) throws RequestExecutionException { - AbstractRangeCommand pageCmd = command.withUpdatedLimit(pageSize); + PartitionRangeReadCommand pageCmd = ((PartitionRangeReadCommand)command).withUpdatedLimit(command.limits().forPaging(pageSize)); if (lastReturnedKey != null) pageCmd = pageCmd.forSubRange(makeExcludingKeyBounds(lastReturnedKey)); - return localQuery - ? pageCmd.executeLocally() - : StorageProxy.getRangeSlice(pageCmd, consistencyLevel); + return pageCmd; } - protected boolean containsPreviousLast(Row first) + protected void recordLast(DecoratedKey key, Row last) { - // When querying the next page, we create a bound that exclude the lastReturnedKey - return false; + lastReturnedKey = key; } - protected boolean recordLast(Row last) - { - lastReturnedKey = last.key; - // We return false as that means "can that last be in the next query?" - return false; - } - - protected boolean isReversed() - { - return false; - } - - private AbstractBounds makeExcludingKeyBounds(RowPosition lastReturnedKey) + private AbstractBounds makeExcludingKeyBounds(PartitionPosition lastReturnedKey) { // We return a range that always exclude lastReturnedKey, since we've already // returned it. - AbstractBounds bounds = command.keyRange; + AbstractBounds bounds = ((PartitionRangeReadCommand)command).dataRange().keyRange(); if (bounds instanceof Range || bounds instanceof Bounds) { - return new Range(lastReturnedKey, bounds.right); + return new Range(lastReturnedKey, bounds.right); } else { - return new ExcludingBounds(lastReturnedKey, bounds.right); + return new ExcludingBounds(lastReturnedKey, bounds.right); } } } diff --git a/src/java/org/apache/cassandra/service/pager/RangeSliceQueryPager.java b/src/java/org/apache/cassandra/service/pager/RangeSliceQueryPager.java index c9a28e8774..6429be0681 100644 --- a/src/java/org/apache/cassandra/service/pager/RangeSliceQueryPager.java +++ b/src/java/org/apache/cassandra/service/pager/RangeSliceQueryPager.java @@ -17,17 +17,16 @@ */ package org.apache.cassandra.service.pager; -import java.util.List; - import org.apache.cassandra.db.*; -import org.apache.cassandra.db.composites.CellName; -import org.apache.cassandra.db.composites.Composite; -import org.apache.cassandra.db.filter.SliceQueryFilter; +import org.apache.cassandra.db.rows.*; +import org.apache.cassandra.db.filter.*; import org.apache.cassandra.dht.*; import org.apache.cassandra.exceptions.RequestExecutionException; -import org.apache.cassandra.service.StorageProxy; import org.apache.cassandra.service.StorageService; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + /** * Pages a RangeSliceCommand whose predicate is a slice query. * @@ -36,27 +35,21 @@ import org.apache.cassandra.service.StorageService; */ public class RangeSliceQueryPager extends AbstractQueryPager { - private final RangeSliceCommand command; + private static final Logger logger = LoggerFactory.getLogger(RangeSliceQueryPager.class); + private volatile DecoratedKey lastReturnedKey; - private volatile CellName lastReturnedName; + private volatile Clustering lastReturnedClustering; - // Don't use directly, use QueryPagers method instead - RangeSliceQueryPager(RangeSliceCommand command, ConsistencyLevel consistencyLevel, boolean localQuery) + public RangeSliceQueryPager(PartitionRangeReadCommand command, PagingState state) { - super(consistencyLevel, command.maxResults, localQuery, command.keyspace, command.columnFamily, command.predicate, command.timestamp); - this.command = command; - assert columnFilter instanceof SliceQueryFilter; - } - - RangeSliceQueryPager(RangeSliceCommand command, ConsistencyLevel consistencyLevel, boolean localQuery, PagingState state) - { - this(command, consistencyLevel, localQuery); + super(command); + assert !command.isNamesQuery(); if (state != null) { lastReturnedKey = StorageService.getPartitioner().decorateKey(state.partitionKey); - lastReturnedName = cfm.comparator.cellFromByteBuffer(state.cellName); - restoreState(state.remaining, true); + lastReturnedClustering = LegacyLayout.decodeClustering(command.metadata(), state.cellName); + restoreState(lastReturnedKey, state.remaining, state.remainingInPartition); } } @@ -64,67 +57,63 @@ public class RangeSliceQueryPager extends AbstractQueryPager { return lastReturnedKey == null ? null - : new PagingState(lastReturnedKey.getKey(), lastReturnedName.toByteBuffer(), maxRemaining()); + : new PagingState(lastReturnedKey.getKey(), LegacyLayout.encodeClustering(command.metadata(), lastReturnedClustering), maxRemaining(), remainingInPartition()); } - protected List queryNextPage(int pageSize, ConsistencyLevel consistencyLevel, boolean localQuery) + protected ReadCommand nextPageReadCommand(int pageSize) throws RequestExecutionException { - SliceQueryFilter sf = (SliceQueryFilter)columnFilter; - AbstractBounds keyRange = lastReturnedKey == null ? command.keyRange : makeIncludingKeyBounds(lastReturnedKey); - Composite start = lastReturnedName == null ? sf.start() : lastReturnedName; - PagedRangeCommand pageCmd = new PagedRangeCommand(command.keyspace, - command.columnFamily, - command.timestamp, - keyRange, - sf, - start, - sf.finish(), - command.rowFilter, - pageSize, - command.countCQL3Rows); - - return localQuery - ? pageCmd.executeLocally() - : StorageProxy.getRangeSlice(pageCmd, consistencyLevel); - } - - protected boolean containsPreviousLast(Row first) - { - if (lastReturnedKey == null || !lastReturnedKey.equals(first.key)) - return false; - - // Same as SliceQueryPager, we ignore a deleted column - Cell firstCell = isReversed() ? lastCell(first.cf) : firstNonStaticCell(first.cf); - return !first.cf.deletionInfo().isDeleted(firstCell) - && firstCell.isLive(timestamp()) - && lastReturnedName.equals(firstCell.name()); - } - - protected boolean recordLast(Row last) - { - lastReturnedKey = last.key; - lastReturnedName = (isReversed() ? firstNonStaticCell(last.cf) : lastCell(last.cf)).name(); - return true; - } - - protected boolean isReversed() - { - return ((SliceQueryFilter)command.predicate).reversed; - } - - private AbstractBounds makeIncludingKeyBounds(RowPosition lastReturnedKey) - { - // We always include lastReturnedKey since we may still be paging within a row, - // and PagedRangeCommand will move over if we're not anyway - AbstractBounds bounds = command.keyRange; - if (bounds instanceof Range || bounds instanceof Bounds) + DataLimits limits; + DataRange fullRange = ((PartitionRangeReadCommand)command).dataRange(); + DataRange pageRange; + if (lastReturnedKey == null) { - return new Bounds(lastReturnedKey, bounds.right); + pageRange = fullRange; + limits = command.limits().forPaging(pageSize); } else { - return new IncludingExcludingBounds(lastReturnedKey, bounds.right); + // We want to include the last returned key only if we haven't achieved our per-partition limit, otherwise, don't bother. + boolean includeLastKey = remainingInPartition() > 0; + AbstractBounds bounds = makeKeyBounds(lastReturnedKey, includeLastKey); + if (includeLastKey) + { + pageRange = fullRange.forPaging(bounds, command.metadata().comparator, lastReturnedClustering, false); + limits = command.limits().forPaging(pageSize, lastReturnedKey.getKey(), remainingInPartition()); + } + else + { + pageRange = fullRange.forSubRange(bounds); + limits = command.limits().forPaging(pageSize); + } + } + + return new PartitionRangeReadCommand(command.metadata(), command.nowInSec(), command.columnFilter(), command.rowFilter(), limits, pageRange); + } + + protected void recordLast(DecoratedKey key, Row last) + { + if (last != null) + { + lastReturnedKey = key; + lastReturnedClustering = last.clustering().takeAlias(); + } + } + + private AbstractBounds makeKeyBounds(PartitionPosition lastReturnedKey, boolean includeLastKey) + { + AbstractBounds bounds = ((PartitionRangeReadCommand)command).dataRange().keyRange(); + if (bounds instanceof Range || bounds instanceof Bounds) + { + return includeLastKey + ? new Bounds(lastReturnedKey, bounds.right) + : new Range(lastReturnedKey, bounds.right); + } + else + { + return includeLastKey + ? new IncludingExcludingBounds(lastReturnedKey, bounds.right) + : new ExcludingBounds(lastReturnedKey, bounds.right); } } } diff --git a/src/java/org/apache/cassandra/service/pager/SinglePartitionPager.java b/src/java/org/apache/cassandra/service/pager/SinglePartitionPager.java index 51bbf90c1b..64886410a2 100644 --- a/src/java/org/apache/cassandra/service/pager/SinglePartitionPager.java +++ b/src/java/org/apache/cassandra/service/pager/SinglePartitionPager.java @@ -19,15 +19,67 @@ package org.apache.cassandra.service.pager; import java.nio.ByteBuffer; -import org.apache.cassandra.db.filter.ColumnCounter; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import org.apache.cassandra.db.*; +import org.apache.cassandra.db.rows.*; +import org.apache.cassandra.db.partitions.*; +import org.apache.cassandra.db.filter.*; +import org.apache.cassandra.exceptions.RequestValidationException; +import org.apache.cassandra.exceptions.RequestExecutionException; +import org.apache.cassandra.service.ClientState; /** * Common interface to single partition queries (by slice and by name). * * For use by MultiPartitionPager. */ -public interface SinglePartitionPager extends QueryPager +public class SinglePartitionPager extends AbstractQueryPager { - public ByteBuffer key(); - public ColumnCounter columnCounter(); + private static final Logger logger = LoggerFactory.getLogger(SinglePartitionPager.class); + + private final SinglePartitionReadCommand command; + + private volatile Clustering lastReturned; + + public SinglePartitionPager(SinglePartitionReadCommand command, PagingState state) + { + super(command); + this.command = command; + + if (state != null) + { + lastReturned = LegacyLayout.decodeClustering(command.metadata(), state.cellName); + restoreState(command.partitionKey(), state.remaining, state.remainingInPartition); + } + } + + public ByteBuffer key() + { + return command.partitionKey().getKey(); + } + + public DataLimits limits() + { + return command.limits(); + } + + public PagingState state() + { + return lastReturned == null + ? null + : new PagingState(null, LegacyLayout.encodeClustering(command.metadata(), lastReturned), maxRemaining(), remainingInPartition()); + } + + protected ReadCommand nextPageReadCommand(int pageSize) + { + return command.forPaging(lastReturned, pageSize); + } + + protected void recordLast(DecoratedKey key, Row last) + { + if (last != null) + lastReturned = last.clustering().takeAlias(); + } } diff --git a/src/java/org/apache/cassandra/service/pager/SliceQueryPager.java b/src/java/org/apache/cassandra/service/pager/SliceQueryPager.java deleted file mode 100644 index bc364aace5..0000000000 --- a/src/java/org/apache/cassandra/service/pager/SliceQueryPager.java +++ /dev/null @@ -1,118 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.apache.cassandra.service.pager; - -import java.nio.ByteBuffer; -import java.util.Collections; -import java.util.List; - -import org.apache.cassandra.db.*; -import org.apache.cassandra.db.composites.Composite; -import org.apache.cassandra.db.filter.SliceQueryFilter; -import org.apache.cassandra.exceptions.RequestValidationException; -import org.apache.cassandra.exceptions.RequestExecutionException; -import org.apache.cassandra.service.ClientState; -import org.apache.cassandra.service.StorageProxy; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; - -/** - * Pager over a SliceFromReadCommand. - */ -public class SliceQueryPager extends AbstractQueryPager implements SinglePartitionPager -{ - private static final Logger logger = LoggerFactory.getLogger(SliceQueryPager.class); - - private final SliceFromReadCommand command; - private final ClientState cstate; - - private volatile Composite lastReturned; - - // Don't use directly, use QueryPagers method instead - SliceQueryPager(SliceFromReadCommand command, ConsistencyLevel consistencyLevel, ClientState cstate, boolean localQuery) - { - super(consistencyLevel, command.filter.count, localQuery, command.ksName, command.cfName, command.filter, command.timestamp); - this.command = command; - this.cstate = cstate; - } - - SliceQueryPager(SliceFromReadCommand command, ConsistencyLevel consistencyLevel, ClientState cstate, boolean localQuery, PagingState state) - { - this(command, consistencyLevel, cstate, localQuery); - - if (state != null) - { - lastReturned = cfm.comparator.fromByteBuffer(state.cellName); - restoreState(state.remaining, true); - } - } - - public ByteBuffer key() - { - return command.key; - } - - public PagingState state() - { - return lastReturned == null - ? null - : new PagingState(null, lastReturned.toByteBuffer(), maxRemaining()); - } - - protected List queryNextPage(int pageSize, ConsistencyLevel consistencyLevel, boolean localQuery) - throws RequestValidationException, RequestExecutionException - { - // For some queries, such as a DISTINCT query on static columns, the limit for slice queries will be lower - // than the page size (in the static example, it will be 1). We use the min here to ensure we don't fetch - // more rows than we're supposed to. See CASSANDRA-8108 for more details. - SliceQueryFilter filter = command.filter.withUpdatedCount(Math.min(command.filter.count, pageSize)); - if (lastReturned != null) - filter = filter.withUpdatedStart(lastReturned, cfm); - - logger.debug("Querying next page of slice query; new filter: {}", filter); - ReadCommand pageCmd = command.withUpdatedFilter(filter); - return localQuery - ? Collections.singletonList(pageCmd.getRow(Keyspace.open(command.ksName))) - : StorageProxy.read(Collections.singletonList(pageCmd), consistencyLevel, cstate); - } - - protected boolean containsPreviousLast(Row first) - { - if (lastReturned == null) - return false; - - Cell firstCell = isReversed() ? lastCell(first.cf) : firstNonStaticCell(first.cf); - // Note: we only return true if the column is the lastReturned *and* it is live. If it is deleted, it is ignored by the - // rest of the paging code (it hasn't been counted as live in particular) and we want to act as if it wasn't there. - return !first.cf.deletionInfo().isDeleted(firstCell) - && firstCell.isLive(timestamp()) - && lastReturned.equals(firstCell.name()); - } - - protected boolean recordLast(Row last) - { - Cell lastCell = isReversed() ? firstNonStaticCell(last.cf) : lastCell(last.cf); - lastReturned = lastCell.name(); - return true; - } - - protected boolean isReversed() - { - return command.filter.reversed; - } -} diff --git a/src/java/org/apache/cassandra/service/paxos/Commit.java b/src/java/org/apache/cassandra/service/paxos/Commit.java index 45d04f983c..6077166d6e 100644 --- a/src/java/org/apache/cassandra/service/paxos/Commit.java +++ b/src/java/org/apache/cassandra/service/paxos/Commit.java @@ -24,14 +24,18 @@ package org.apache.cassandra.service.paxos; import java.io.DataInput; import java.io.IOException; import java.util.UUID; -import java.nio.ByteBuffer; import com.google.common.base.Objects; import org.apache.cassandra.config.CFMetaData; +import org.apache.cassandra.config.ColumnDefinition; import org.apache.cassandra.db.*; +import org.apache.cassandra.db.rows.*; +import org.apache.cassandra.db.partitions.PartitionUpdate; import org.apache.cassandra.io.IVersionedSerializer; import org.apache.cassandra.io.util.DataOutputPlus; +import org.apache.cassandra.net.MessagingService; +import org.apache.cassandra.service.StorageService; import org.apache.cassandra.utils.ByteBufferUtil; import org.apache.cassandra.utils.UUIDGen; import org.apache.cassandra.utils.UUIDSerializer; @@ -40,34 +44,31 @@ public class Commit { public static final CommitSerializer serializer = new CommitSerializer(); - public final ByteBuffer key; public final UUID ballot; - public final ColumnFamily update; + public final PartitionUpdate update; - public Commit(ByteBuffer key, UUID ballot, ColumnFamily update) + public Commit(UUID ballot, PartitionUpdate update) { - assert key != null; assert ballot != null; assert update != null; - this.key = key; this.ballot = ballot; this.update = update; } - public static Commit newPrepare(ByteBuffer key, CFMetaData metadata, UUID ballot) + public static Commit newPrepare(DecoratedKey key, CFMetaData metadata, UUID ballot) { - return new Commit(key, ballot, ArrayBackedSortedColumns.factory.create(metadata)); + return new Commit(ballot, PartitionUpdate.emptyUpdate(metadata, key)); } - public static Commit newProposal(ByteBuffer key, UUID ballot, ColumnFamily update) + public static Commit newProposal(UUID ballot, PartitionUpdate update) { - return new Commit(key, ballot, updatesWithPaxosTime(update, ballot)); + return new Commit(ballot, updatesWithPaxosTime(update, ballot)); } - public static Commit emptyCommit(ByteBuffer key, CFMetaData metadata) + public static Commit emptyCommit(DecoratedKey key, CFMetaData metadata) { - return new Commit(key, UUIDGen.minTimeUUID(0), ArrayBackedSortedColumns.factory.create(metadata)); + return new Commit(UUIDGen.minTimeUUID(0), PartitionUpdate.emptyUpdate(metadata, key)); } public boolean isAfter(Commit other) @@ -83,7 +84,7 @@ public class Commit public Mutation makeMutation() { assert update != null; - return new Mutation(key, update); + return new Mutation(update); } @Override @@ -95,7 +96,6 @@ public class Commit Commit commit = (Commit) o; if (!ballot.equals(commit.ballot)) return false; - if (!key.equals(commit.key)) return false; if (!update.equals(commit.update)) return false; return true; @@ -104,52 +104,88 @@ public class Commit @Override public int hashCode() { - return Objects.hashCode(key, ballot, update); + return Objects.hashCode(ballot, update); } - private static ColumnFamily updatesWithPaxosTime(ColumnFamily updates, UUID ballot) + private static PartitionUpdate updatesWithPaxosTime(PartitionUpdate update, UUID ballot) { - ColumnFamily cf = updates.cloneMeShallow(); long t = UUIDGen.microsTimestamp(ballot); - // For the tombstones, we use t-1 so that when insert a collection literall, the range tombstone that deletes the previous values of - // the collection and we want that to have a lower timestamp and our new values. Since tombstones wins over normal insert, using t-1 - // should not be a problem in general (see #6069). - cf.deletionInfo().updateAllTimestamp(t-1); - for (Cell cell : updates) - cf.addAtom(cell.withUpdatedTimestamp(t)); - return cf; + // Using t-1 for tombstones so deletion doesn't trump newly inserted data (#6069) + PartitionUpdate newUpdate = new PartitionUpdate(update.metadata(), + update.partitionKey(), + update.deletionInfo().updateAllTimestamp(t-1), + update.columns(), + update.rowCount()); + + if (!update.staticRow().isEmpty()) + copyWithUpdatedTimestamp(update.staticRow(), newUpdate.staticWriter(), t); + + for (Row row : update) + copyWithUpdatedTimestamp(row, newUpdate.writer(), t); + + return newUpdate; + } + + private static void copyWithUpdatedTimestamp(Row row, Row.Writer writer, long timestamp) + { + Rows.writeClustering(row.clustering(), writer); + writer.writePartitionKeyLivenessInfo(row.primaryKeyLivenessInfo().withUpdatedTimestamp(timestamp)); + writer.writeRowDeletion(row.deletion()); + + for (Cell cell : row) + writer.writeCell(cell.column(), cell.isCounterCell(), cell.value(), cell.livenessInfo().withUpdatedTimestamp(timestamp), cell.path()); + + for (int i = 0; i < row.columns().complexColumnCount(); i++) + { + ColumnDefinition c = row.columns().getComplex(i); + DeletionTime dt = row.getDeletion(c); + // We use t-1 to make sure that on inserting a collection literal, the deletion that comes with it does not + // end up deleting the inserted data (see #6069) + if (!dt.isLive()) + writer.writeComplexDeletion(c, new SimpleDeletionTime(timestamp-1, dt.localDeletionTime())); + } + writer.endOfRow(); } @Override public String toString() { - return String.format("Commit(%s, %s, %s)", ByteBufferUtil.bytesToHex(key), ballot, update); + return String.format("Commit(%s, %s)", ballot, update); } public static class CommitSerializer implements IVersionedSerializer { public void serialize(Commit commit, DataOutputPlus out, int version) throws IOException { - ByteBufferUtil.writeWithShortLength(commit.key, out); + if (version < MessagingService.VERSION_30) + ByteBufferUtil.writeWithShortLength(commit.update.partitionKey().getKey(), out); + UUIDSerializer.serializer.serialize(commit.ballot, out, version); - ColumnFamily.serializer.serialize(commit.update, out, version); + PartitionUpdate.serializer.serialize(commit.update, out, version); } public Commit deserialize(DataInput in, int version) throws IOException { - return new Commit(ByteBufferUtil.readWithShortLength(in), - UUIDSerializer.serializer.deserialize(in, version), - ColumnFamily.serializer.deserialize(in, - ArrayBackedSortedColumns.factory, - ColumnSerializer.Flag.LOCAL, - version)); + DecoratedKey key = null; + if (version < MessagingService.VERSION_30) + key = StorageService.getPartitioner().decorateKey(ByteBufferUtil.readWithShortLength(in)); + + UUID ballot = UUIDSerializer.serializer.deserialize(in, version); + PartitionUpdate update = PartitionUpdate.serializer.deserialize(in, version, SerializationHelper.Flag.LOCAL, key); + return new Commit(ballot, update); } public long serializedSize(Commit commit, int version) { - return 2 + commit.key.remaining() - + UUIDSerializer.serializer.serializedSize(commit.ballot, version) - + ColumnFamily.serializer.serializedSize(commit.update, version); + TypeSizes sizes = TypeSizes.NATIVE; + + int size = 0; + if (version < MessagingService.VERSION_30) + size += ByteBufferUtil.serializedSizeWithShortLength(commit.update.partitionKey().getKey(), sizes); + + return size + + UUIDSerializer.serializer.serializedSize(commit.ballot, version) + + PartitionUpdate.serializer.serializedSize(commit.update, version, sizes); } } } diff --git a/src/java/org/apache/cassandra/service/paxos/PaxosState.java b/src/java/org/apache/cassandra/service/paxos/PaxosState.java index 01e03f47c6..20ccb90255 100644 --- a/src/java/org/apache/cassandra/service/paxos/PaxosState.java +++ b/src/java/org/apache/cassandra/service/paxos/PaxosState.java @@ -39,14 +39,14 @@ public class PaxosState private final Commit accepted; private final Commit mostRecentCommit; - public PaxosState(ByteBuffer key, CFMetaData metadata) + public PaxosState(DecoratedKey key, CFMetaData metadata) { this(Commit.emptyCommit(key, metadata), Commit.emptyCommit(key, metadata), Commit.emptyCommit(key, metadata)); } public PaxosState(Commit promised, Commit accepted, Commit mostRecentCommit) { - assert promised.key == accepted.key && accepted.key == mostRecentCommit.key; + assert promised.update.partitionKey().equals(accepted.update.partitionKey()) && accepted.update.partitionKey().equals(mostRecentCommit.update.partitionKey()); assert promised.update.metadata() == accepted.update.metadata() && accepted.update.metadata() == mostRecentCommit.update.metadata(); this.promised = promised; @@ -59,11 +59,11 @@ public class PaxosState long start = System.nanoTime(); try { - Lock lock = LOCKS.get(toPrepare.key); + Lock lock = LOCKS.get(toPrepare.update.partitionKey()); lock.lock(); try { - PaxosState state = SystemKeyspace.loadPaxosState(toPrepare.key, toPrepare.update.metadata()); + PaxosState state = SystemKeyspace.loadPaxosState(toPrepare.update.partitionKey(), toPrepare.update.metadata()); if (toPrepare.isAfter(state.promised)) { Tracing.trace("Promising ballot {}", toPrepare.ballot); @@ -94,11 +94,11 @@ public class PaxosState long start = System.nanoTime(); try { - Lock lock = LOCKS.get(proposal.key); + Lock lock = LOCKS.get(proposal.update.partitionKey()); lock.lock(); try { - PaxosState state = SystemKeyspace.loadPaxosState(proposal.key, proposal.update.metadata()); + PaxosState state = SystemKeyspace.loadPaxosState(proposal.update.partitionKey(), proposal.update.metadata()); if (proposal.hasBallot(state.promised.ballot) || proposal.isAfter(state.promised)) { Tracing.trace("Accepting proposal {}", proposal); diff --git a/src/java/org/apache/cassandra/service/paxos/PrepareCallback.java b/src/java/org/apache/cassandra/service/paxos/PrepareCallback.java index a446b0b6c6..7b5edf2e31 100644 --- a/src/java/org/apache/cassandra/service/paxos/PrepareCallback.java +++ b/src/java/org/apache/cassandra/service/paxos/PrepareCallback.java @@ -29,6 +29,7 @@ import java.util.concurrent.ConcurrentHashMap; import com.google.common.base.Predicate; import com.google.common.collect.Iterables; import org.apache.cassandra.db.ConsistencyLevel; +import org.apache.cassandra.db.DecoratedKey; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -46,7 +47,7 @@ public class PrepareCallback extends AbstractPaxosCallback private final Map commitsByReplica = new ConcurrentHashMap(); - public PrepareCallback(ByteBuffer key, CFMetaData metadata, int targets, ConsistencyLevel consistency) + public PrepareCallback(DecoratedKey key, CFMetaData metadata, int targets, ConsistencyLevel consistency) { super(targets, consistency); // need to inject the right key in the empty commit so comparing with empty commits in the reply works as expected diff --git a/src/java/org/apache/cassandra/service/paxos/PrepareResponse.java b/src/java/org/apache/cassandra/service/paxos/PrepareResponse.java index e766e3417a..bf0740233f 100644 --- a/src/java/org/apache/cassandra/service/paxos/PrepareResponse.java +++ b/src/java/org/apache/cassandra/service/paxos/PrepareResponse.java @@ -23,14 +23,14 @@ package org.apache.cassandra.service.paxos; import java.io.DataInput; import java.io.IOException; -import java.nio.ByteBuffer; +import java.util.UUID; -import org.apache.cassandra.db.ArrayBackedSortedColumns; -import org.apache.cassandra.db.ColumnFamily; -import org.apache.cassandra.db.ColumnSerializer; +import org.apache.cassandra.db.TypeSizes; +import org.apache.cassandra.db.rows.SerializationHelper; +import org.apache.cassandra.db.partitions.PartitionUpdate; import org.apache.cassandra.io.IVersionedSerializer; import org.apache.cassandra.io.util.DataOutputPlus; -import org.apache.cassandra.utils.ByteBufferUtil; +import org.apache.cassandra.net.MessagingService; import org.apache.cassandra.utils.UUIDSerializer; public class PrepareResponse @@ -49,7 +49,7 @@ public class PrepareResponse public PrepareResponse(boolean promised, Commit inProgressCommit, Commit mostRecentCommit) { - assert inProgressCommit.key == mostRecentCommit.key; + assert inProgressCommit.update.partitionKey().equals(mostRecentCommit.update.partitionKey()); assert inProgressCommit.update.metadata() == mostRecentCommit.update.metadata(); this.promised = promised; @@ -68,38 +68,53 @@ public class PrepareResponse public void serialize(PrepareResponse response, DataOutputPlus out, int version) throws IOException { out.writeBoolean(response.promised); - ByteBufferUtil.writeWithShortLength(response.inProgressCommit.key, out); - UUIDSerializer.serializer.serialize(response.inProgressCommit.ballot, out, version); - ColumnFamily.serializer.serialize(response.inProgressCommit.update, out, version); - UUIDSerializer.serializer.serialize(response.mostRecentCommit.ballot, out, version); - ColumnFamily.serializer.serialize(response.mostRecentCommit.update, out, version); + Commit.serializer.serialize(response.inProgressCommit, out, version); + + if (version < MessagingService.VERSION_30) + { + UUIDSerializer.serializer.serialize(response.mostRecentCommit.ballot, out, version); + PartitionUpdate.serializer.serialize(response.mostRecentCommit.update, out, version); + } + else + { + Commit.serializer.serialize(response.mostRecentCommit, out, version); + } } public PrepareResponse deserialize(DataInput in, int version) throws IOException { boolean success = in.readBoolean(); - ByteBuffer key = ByteBufferUtil.readWithShortLength(in); - return new PrepareResponse(success, - new Commit(key, - UUIDSerializer.serializer.deserialize(in, version), - ColumnFamily.serializer.deserialize(in, - ArrayBackedSortedColumns.factory, - ColumnSerializer.Flag.LOCAL, version)), - new Commit(key, - UUIDSerializer.serializer.deserialize(in, version), - ColumnFamily.serializer.deserialize(in, - ArrayBackedSortedColumns.factory, - ColumnSerializer.Flag.LOCAL, version))); + Commit inProgress = Commit.serializer.deserialize(in, version); + Commit mostRecent; + if (version < MessagingService.VERSION_30) + { + UUID ballot = UUIDSerializer.serializer.deserialize(in, version); + PartitionUpdate update = PartitionUpdate.serializer.deserialize(in, version, SerializationHelper.Flag.LOCAL, inProgress.update.partitionKey()); + mostRecent = new Commit(ballot, update); + } + else + { + mostRecent = Commit.serializer.deserialize(in, version); + } + return new PrepareResponse(success, inProgress, mostRecent); } public long serializedSize(PrepareResponse response, int version) { - return 1 - + 2 + response.inProgressCommit.key.remaining() - + UUIDSerializer.serializer.serializedSize(response.inProgressCommit.ballot, version) - + ColumnFamily.serializer.serializedSize(response.inProgressCommit.update, version) - + UUIDSerializer.serializer.serializedSize(response.mostRecentCommit.ballot, version) - + ColumnFamily.serializer.serializedSize(response.mostRecentCommit.update, version); + TypeSizes sizes = TypeSizes.NATIVE; + long size = sizes.sizeof(response.promised) + + Commit.serializer.serializedSize(response.inProgressCommit, version); + + if (version < MessagingService.VERSION_30) + { + size += UUIDSerializer.serializer.serializedSize(response.mostRecentCommit.ballot, version); + size += PartitionUpdate.serializer.serializedSize(response.mostRecentCommit.update, version, sizes); + } + else + { + size += Commit.serializer.serializedSize(response.mostRecentCommit, version); + } + return size; } } } diff --git a/src/java/org/apache/cassandra/streaming/StreamReader.java b/src/java/org/apache/cassandra/streaming/StreamReader.java index 1a3980d71d..66eb220cca 100644 --- a/src/java/org/apache/cassandra/streaming/StreamReader.java +++ b/src/java/org/apache/cassandra/streaming/StreamReader.java @@ -18,31 +18,35 @@ package org.apache.cassandra.streaming; import java.io.*; +import java.nio.ByteBuffer; import java.nio.channels.Channels; import java.nio.channels.ReadableByteChannel; import java.util.Collection; import java.util.UUID; import com.google.common.base.Throwables; -import org.apache.cassandra.io.sstable.format.SSTableFormat; -import org.apache.cassandra.io.sstable.format.SSTableWriter; -import org.apache.cassandra.io.sstable.format.Version; +import com.google.common.collect.UnmodifiableIterator; import org.slf4j.Logger; import org.slf4j.LoggerFactory; import com.ning.compress.lzf.LZFInputStream; +import org.apache.cassandra.config.CFMetaData; import org.apache.cassandra.config.Schema; -import org.apache.cassandra.db.ColumnFamilyStore; -import org.apache.cassandra.db.DecoratedKey; -import org.apache.cassandra.db.Directories; -import org.apache.cassandra.db.Keyspace; +import org.apache.cassandra.db.*; +import org.apache.cassandra.db.rows.*; +import org.apache.cassandra.db.context.CounterContext; import org.apache.cassandra.io.sstable.Descriptor; +import org.apache.cassandra.io.sstable.SSTableSimpleIterator; +import org.apache.cassandra.io.sstable.format.SSTableFormat; +import org.apache.cassandra.io.sstable.format.SSTableWriter; +import org.apache.cassandra.io.sstable.format.Version; import org.apache.cassandra.service.StorageService; import org.apache.cassandra.streaming.messages.FileMessageHeader; import org.apache.cassandra.utils.ByteBufferUtil; import org.apache.cassandra.utils.BytesReadTracker; +import org.apache.cassandra.utils.FBUtilities; import org.apache.cassandra.utils.Pair; @@ -60,6 +64,7 @@ public class StreamReader protected final long repairedAt; protected final SSTableFormat.Type format; protected final int sstableLevel; + protected final SerializationHeader.Component header; protected Descriptor desc; @@ -69,10 +74,11 @@ public class StreamReader this.cfId = header.cfId; this.estimatedKeys = header.estimatedKeys; this.sections = header.sections; - this.inputVersion = header.format.info.getVersion(header.version); + this.inputVersion = header.version; this.repairedAt = header.repairedAt; this.format = header.format; this.sstableLevel = header.sstableLevel; + this.header = header.header; } /** @@ -98,17 +104,18 @@ public class StreamReader DataInputStream dis = new DataInputStream(new LZFInputStream(Channels.newInputStream(channel))); BytesReadTracker in = new BytesReadTracker(dis); + StreamDeserializer deserializer = new StreamDeserializer(cfs.metadata, in, inputVersion, header.toHeader(cfs.metadata)); try { while (in.getBytesRead() < totalSize) { - writeRow(writer, in, cfs); - + writePartition(deserializer, writer, cfs); // TODO move this to BytesReadTracker session.progress(desc, ProgressInfo.Direction.IN, in.getBytesRead(), totalSize); } return writer; - } catch (Throwable e) + } + catch (Throwable e) { writer.abort(); drain(dis, in.getBytesRead()); @@ -126,7 +133,7 @@ public class StreamReader throw new IOException("Insufficient disk space to store " + totalSize + " bytes"); desc = Descriptor.fromFilename(cfs.getTempSSTablePath(cfs.directories.getLocationForDisk(localDir), format)); - return SSTableWriter.create(desc, estimatedKeys, repairedAt, sstableLevel); + return SSTableWriter.create(desc, estimatedKeys, repairedAt, sstableLevel, header.toHeader(cfs.metadata)); } protected void drain(InputStream dis, long bytesRead) throws IOException @@ -156,10 +163,141 @@ public class StreamReader return size; } - protected void writeRow(SSTableWriter writer, DataInput in, ColumnFamilyStore cfs) throws IOException + protected void writePartition(StreamDeserializer deserializer, SSTableWriter writer, ColumnFamilyStore cfs) throws IOException { - DecoratedKey key = StorageService.getPartitioner().decorateKey(ByteBufferUtil.readWithShortLength(in)); - writer.appendFromStream(key, cfs.metadata, in, inputVersion); - cfs.invalidateCachedRow(key); + DecoratedKey key = deserializer.newPartition(); + writer.append(deserializer); + deserializer.checkForExceptions(); + cfs.invalidateCachedPartition(key); + } + + public static class StreamDeserializer extends UnmodifiableIterator implements UnfilteredRowIterator + { + private final CFMetaData metadata; + private final DataInput in; + private final SerializationHeader header; + private final SerializationHelper helper; + + private DecoratedKey key; + private DeletionTime partitionLevelDeletion; + private SSTableSimpleIterator iterator; + private Row staticRow; + private IOException exception; + + private final CounterFilteredRow counterRow; + + public StreamDeserializer(CFMetaData metadata, DataInput in, Version version, SerializationHeader header) + { + assert version.storeRows() : "We don't allow streaming from pre-3.0 nodes"; + this.metadata = metadata; + this.in = in; + this.helper = new SerializationHelper(version.correspondingMessagingVersion(), SerializationHelper.Flag.PRESERVE_SIZE); + this.header = header; + this.counterRow = metadata.isCounter() ? new CounterFilteredRow() : null; + } + + public DecoratedKey newPartition() throws IOException + { + key = StorageService.getPartitioner().decorateKey(ByteBufferUtil.readWithShortLength(in)); + partitionLevelDeletion = DeletionTime.serializer.deserialize(in); + iterator = SSTableSimpleIterator.create(metadata, in, header, helper, partitionLevelDeletion); + staticRow = iterator.readStaticRow(); + return key; + } + + public CFMetaData metadata() + { + return metadata; + } + + public PartitionColumns columns() + { + // We don't know which columns we'll get so assume it can be all of them + return metadata.partitionColumns(); + } + + public boolean isReverseOrder() + { + return false; + } + + public DecoratedKey partitionKey() + { + return key; + } + + public DeletionTime partitionLevelDeletion() + { + return partitionLevelDeletion; + } + + public Row staticRow() + { + return staticRow; + } + + public RowStats stats() + { + return header.stats(); + } + + public boolean hasNext() + { + try + { + return iterator.hasNext(); + } + catch (IOError e) + { + if (e.getCause() != null && e.getCause() instanceof IOException) + { + exception = (IOException)e.getCause(); + return false; + } + throw e; + } + } + + public Unfiltered next() + { + // Note that in practice we know that IOException will be thrown by hasNext(), because that's + // where the actual reading happens, so we don't bother catching RuntimeException here (contrarily + // to what we do in hasNext) + Unfiltered unfiltered = iterator.next(); + return metadata.isCounter() && unfiltered.kind() == Unfiltered.Kind.ROW + ? maybeMarkLocalToBeCleared((Row) unfiltered) + : unfiltered; + } + + private Row maybeMarkLocalToBeCleared(Row row) + { + return metadata.isCounter() + ? counterRow.setTo(row) + : row; + } + + public void checkForExceptions() throws IOException + { + if (exception != null) + throw exception; + } + + public void close() + { + } + } + + private static class CounterFilteredRow extends WrappingRow + { + protected Cell filterCell(Cell cell) + { + if (!cell.isCounterCell()) + return cell; + + ByteBuffer marked = CounterContext.instance().markLocalToBeCleared(cell.value()); + return marked == cell.value() + ? cell + : Cells.create(cell.column(), true, marked, cell.livenessInfo(), cell.path()); + } } } diff --git a/src/java/org/apache/cassandra/streaming/StreamSession.java b/src/java/org/apache/cassandra/streaming/StreamSession.java index 44522db3db..d27c4e23f0 100644 --- a/src/java/org/apache/cassandra/streaming/StreamSession.java +++ b/src/java/org/apache/cassandra/streaming/StreamSession.java @@ -24,8 +24,6 @@ import java.util.*; import java.util.concurrent.*; import java.util.concurrent.atomic.AtomicBoolean; -import javax.annotation.Nullable; - import com.google.common.base.Function; import com.google.common.collect.*; @@ -37,7 +35,7 @@ import org.slf4j.LoggerFactory; import org.apache.cassandra.config.DatabaseDescriptor; import org.apache.cassandra.db.ColumnFamilyStore; import org.apache.cassandra.db.Keyspace; -import org.apache.cassandra.db.RowPosition; +import org.apache.cassandra.db.PartitionPosition; import org.apache.cassandra.dht.AbstractBounds; import org.apache.cassandra.dht.Range; import org.apache.cassandra.dht.Token; @@ -316,7 +314,7 @@ public class StreamSession implements IEndpointStateChangeSubscriber { for (ColumnFamilyStore cfStore : stores) { - final List> rowBoundsList = new ArrayList<>(ranges.size()); + final List> rowBoundsList = new ArrayList<>(ranges.size()); for (Range range : ranges) rowBoundsList.add(Range.makeRowRange(range)); refs.addAll(cfStore.selectAndReference(new Function>() @@ -327,7 +325,7 @@ public class StreamSession implements IEndpointStateChangeSubscriber Set sstables = Sets.newHashSet(); if (filteredSSTables != null) { - for (AbstractBounds rowBounds : rowBoundsList) + for (AbstractBounds rowBounds : rowBoundsList) { // sstableInBounds may contain early opened sstables for (SSTableReader sstable : view.sstablesInBounds(rowBounds)) diff --git a/src/java/org/apache/cassandra/streaming/compress/CompressedStreamReader.java b/src/java/org/apache/cassandra/streaming/compress/CompressedStreamReader.java index 1936a94fc1..47832f06ab 100644 --- a/src/java/org/apache/cassandra/streaming/compress/CompressedStreamReader.java +++ b/src/java/org/apache/cassandra/streaming/compress/CompressedStreamReader.java @@ -79,6 +79,7 @@ public class CompressedStreamReader extends StreamReader CompressedInputStream cis = new CompressedInputStream(Channels.newInputStream(channel), compressionInfo); BytesReadTracker in = new BytesReadTracker(new DataInputStream(cis)); + StreamDeserializer deserializer = new StreamDeserializer(cfs.metadata, in, inputVersion, header.toHeader(cfs.metadata)); try { for (Pair section : sections) @@ -92,8 +93,7 @@ public class CompressedStreamReader extends StreamReader while (in.getBytesRead() < sectionLength) { - writeRow(writer, in, cfs); - + writePartition(deserializer, writer, cfs); // when compressed, report total bytes of compressed chunks read since remoteFile.size is the sum of chunks transferred session.progress(desc, ProgressInfo.Direction.IN, cis.getTotalCompressedBytesRead(), totalSize); } diff --git a/src/java/org/apache/cassandra/streaming/messages/FileMessageHeader.java b/src/java/org/apache/cassandra/streaming/messages/FileMessageHeader.java index e9c99fed8e..b8e7979ec9 100644 --- a/src/java/org/apache/cassandra/streaming/messages/FileMessageHeader.java +++ b/src/java/org/apache/cassandra/streaming/messages/FileMessageHeader.java @@ -23,11 +23,15 @@ import java.util.ArrayList; import java.util.List; import java.util.UUID; +import org.apache.cassandra.config.DatabaseDescriptor; +import org.apache.cassandra.config.Schema; +import org.apache.cassandra.db.SerializationHeader; import org.apache.cassandra.db.TypeSizes; import org.apache.cassandra.io.IVersionedSerializer; import org.apache.cassandra.io.compress.CompressionMetadata; import org.apache.cassandra.io.sstable.format.SSTableFormat; import org.apache.cassandra.io.util.DataOutputPlus; +import org.apache.cassandra.io.sstable.format.Version; import org.apache.cassandra.net.MessagingService; import org.apache.cassandra.streaming.compress.CompressionInfo; import org.apache.cassandra.utils.Pair; @@ -43,7 +47,7 @@ public class FileMessageHeader public final UUID cfId; public final int sequenceNumber; /** SSTable version */ - public final String version; + public final Version version; /** SSTable format **/ public final SSTableFormat.Type format; @@ -52,16 +56,18 @@ public class FileMessageHeader public final CompressionInfo compressionInfo; public final long repairedAt; public final int sstableLevel; + public final SerializationHeader.Component header; public FileMessageHeader(UUID cfId, int sequenceNumber, - String version, + Version version, SSTableFormat.Type format, long estimatedKeys, List> sections, CompressionInfo compressionInfo, long repairedAt, - int sstableLevel) + int sstableLevel, + SerializationHeader.Component header) { this.cfId = cfId; this.sequenceNumber = sequenceNumber; @@ -72,6 +78,7 @@ public class FileMessageHeader this.compressionInfo = compressionInfo; this.repairedAt = repairedAt; this.sstableLevel = sstableLevel; + this.header = header; } /** @@ -134,7 +141,7 @@ public class FileMessageHeader { UUIDSerializer.serializer.serialize(header.cfId, out, version); out.writeInt(header.sequenceNumber); - out.writeUTF(header.version); + out.writeUTF(header.version.toString()); //We can't stream to a node that doesn't understand a new sstable format if (version < StreamMessage.VERSION_22 && header.format != SSTableFormat.Type.LEGACY && header.format != SSTableFormat.Type.BIG) @@ -153,13 +160,16 @@ public class FileMessageHeader CompressionInfo.serializer.serialize(header.compressionInfo, out, version); out.writeLong(header.repairedAt); out.writeInt(header.sstableLevel); + + if (version >= StreamMessage.VERSION_30) + SerializationHeader.serializer.serialize(header.header, out); } public FileMessageHeader deserialize(DataInput in, int version) throws IOException { UUID cfId = UUIDSerializer.serializer.deserialize(in, MessagingService.current_version); int sequenceNumber = in.readInt(); - String sstableVersion = in.readUTF(); + Version sstableVersion = DatabaseDescriptor.getSSTableFormat().info.getVersion(in.readUTF()); SSTableFormat.Type format = SSTableFormat.Type.LEGACY; if (version >= StreamMessage.VERSION_22) @@ -173,14 +183,18 @@ public class FileMessageHeader CompressionInfo compressionInfo = CompressionInfo.serializer.deserialize(in, MessagingService.current_version); long repairedAt = in.readLong(); int sstableLevel = in.readInt(); - return new FileMessageHeader(cfId, sequenceNumber, sstableVersion, format, estimatedKeys, sections, compressionInfo, repairedAt, sstableLevel); + SerializationHeader.Component header = version >= StreamMessage.VERSION_30 + ? SerializationHeader.serializer.deserialize(sstableVersion, in) + : null; + + return new FileMessageHeader(cfId, sequenceNumber, sstableVersion, format, estimatedKeys, sections, compressionInfo, repairedAt, sstableLevel, header); } public long serializedSize(FileMessageHeader header, int version) { long size = UUIDSerializer.serializer.serializedSize(header.cfId, version); size += TypeSizes.NATIVE.sizeof(header.sequenceNumber); - size += TypeSizes.NATIVE.sizeof(header.version); + size += TypeSizes.NATIVE.sizeof(header.version.toString()); if (version >= StreamMessage.VERSION_22) size += TypeSizes.NATIVE.sizeof(header.format.name); @@ -195,6 +209,10 @@ public class FileMessageHeader } size += CompressionInfo.serializer.serializedSize(header.compressionInfo, version); size += TypeSizes.NATIVE.sizeof(header.sstableLevel); + + if (version >= StreamMessage.VERSION_30) + size += SerializationHeader.serializer.serializedSize(header.header); + return size; } } diff --git a/src/java/org/apache/cassandra/streaming/messages/OutgoingFileMessage.java b/src/java/org/apache/cassandra/streaming/messages/OutgoingFileMessage.java index 5b34bd80f8..82e662032f 100644 --- a/src/java/org/apache/cassandra/streaming/messages/OutgoingFileMessage.java +++ b/src/java/org/apache/cassandra/streaming/messages/OutgoingFileMessage.java @@ -70,13 +70,14 @@ public class OutgoingFileMessage extends StreamMessage } this.header = new FileMessageHeader(sstable.metadata.cfId, sequenceNumber, - sstable.descriptor.version.toString(), + sstable.descriptor.version, sstable.descriptor.formatType, estimatedKeys, sections, compressionInfo, repairedAt, - keepSSTableLevel ? sstable.getSSTableLevel() : 0); + keepSSTableLevel ? sstable.getSSTableLevel() : 0, + sstable.header == null ? null : sstable.header.toComponent()); } public synchronized void serialize(DataOutputStreamPlus out, int version, StreamSession session) throws IOException diff --git a/src/java/org/apache/cassandra/streaming/messages/StreamMessage.java b/src/java/org/apache/cassandra/streaming/messages/StreamMessage.java index d4e8a81ac9..3db2dbf87c 100644 --- a/src/java/org/apache/cassandra/streaming/messages/StreamMessage.java +++ b/src/java/org/apache/cassandra/streaming/messages/StreamMessage.java @@ -34,7 +34,8 @@ public abstract class StreamMessage /** Streaming protocol version */ public static final int VERSION_20 = 2; public static final int VERSION_22 = 3; - public static final int CURRENT_VERSION = VERSION_22; + public static final int VERSION_30 = 4; + public static final int CURRENT_VERSION = VERSION_30; public static void serialize(StreamMessage message, DataOutputStreamPlus out, int version, StreamSession session) throws IOException { diff --git a/src/java/org/apache/cassandra/thrift/CassandraServer.java b/src/java/org/apache/cassandra/thrift/CassandraServer.java index 04d3d13033..d6f0757a78 100644 --- a/src/java/org/apache/cassandra/thrift/CassandraServer.java +++ b/src/java/org/apache/cassandra/thrift/CassandraServer.java @@ -39,10 +39,11 @@ import org.apache.cassandra.config.*; import org.apache.cassandra.cql3.QueryOptions; import org.apache.cassandra.cql3.statements.ParsedStatement; import org.apache.cassandra.db.*; -import org.apache.cassandra.db.composites.*; +import org.apache.cassandra.db.rows.*; +import org.apache.cassandra.db.partitions.*; import org.apache.cassandra.db.context.CounterContext; -import org.apache.cassandra.db.filter.ColumnSlice; import org.apache.cassandra.db.filter.*; +import org.apache.cassandra.db.marshal.AbstractType; import org.apache.cassandra.db.marshal.TimeUUIDType; import org.apache.cassandra.dht.*; import org.apache.cassandra.dht.Range; @@ -55,9 +56,7 @@ import org.apache.cassandra.serializers.MarshalException; import org.apache.cassandra.service.*; import org.apache.cassandra.service.pager.QueryPagers; import org.apache.cassandra.tracing.Tracing; -import org.apache.cassandra.utils.ByteBufferUtil; -import org.apache.cassandra.utils.Pair; -import org.apache.cassandra.utils.UUIDGen; +import org.apache.cassandra.utils.*; import org.apache.thrift.TException; public class CassandraServer implements Cassandra.Iface @@ -84,19 +83,15 @@ public class CassandraServer implements Cassandra.Iface return ThriftSessionManager.instance.currentSession(); } - protected Map readColumnFamily(List commands, org.apache.cassandra.db.ConsistencyLevel consistency_level, ClientState cState) + protected PartitionIterator read(List> commands, org.apache.cassandra.db.ConsistencyLevel consistency_level, ClientState cState) throws org.apache.cassandra.exceptions.InvalidRequestException, UnavailableException, TimedOutException { - // TODO - Support multiple column families per row, right now row only contains 1 column family - Map columnFamilyKeyMap = new HashMap(); - - List rows = null; try { schedule(DatabaseDescriptor.getReadRpcTimeout()); try { - rows = StorageProxy.read(commands, consistency_level, cState); + return StorageProxy.read(new SinglePartitionReadCommand.Group(commands, DataLimits.NONE), consistency_level, cState); } finally { @@ -105,180 +100,176 @@ public class CassandraServer implements Cassandra.Iface } catch (RequestExecutionException e) { - ThriftConversion.rethrow(e); + throw ThriftConversion.rethrow(e); } - - for (Row row: rows) - { - columnFamilyKeyMap.put(row.key, row.cf); - } - return columnFamilyKeyMap; } - public List thriftifyColumns(Collection cells, boolean reverseOrder, long now) + public List thriftifyColumns(CFMetaData metadata, Iterator cells) { - ArrayList thriftColumns = new ArrayList(cells.size()); - for (Cell cell : cells) + ArrayList thriftColumns = new ArrayList(); + while (cells.hasNext()) { - if (!cell.isLive(now)) - continue; - - thriftColumns.add(thriftifyColumnWithName(cell, cell.name().toByteBuffer())); + LegacyLayout.LegacyCell cell = cells.next(); + thriftColumns.add(thriftifyColumnWithName(metadata, cell, cell.name.encode(metadata))); } - - // we have to do the reversing here, since internally we pass results around in ColumnFamily - // objects, which always sort their cells in the "natural" order - // TODO this is inconvenient for direct users of StorageProxy - if (reverseOrder) - Collections.reverse(thriftColumns); return thriftColumns; } - private ColumnOrSuperColumn thriftifyColumnWithName(Cell cell, ByteBuffer newName) + private ColumnOrSuperColumn thriftifyColumnWithName(CFMetaData metadata, LegacyLayout.LegacyCell cell, ByteBuffer newName) { - if (cell instanceof CounterCell) - return new ColumnOrSuperColumn().setCounter_column(thriftifySubCounter(cell).setName(newName)); + if (cell.isCounter()) + return new ColumnOrSuperColumn().setCounter_column(thriftifySubCounter(metadata, cell).setName(newName)); else - return new ColumnOrSuperColumn().setColumn(thriftifySubColumn(cell).setName(newName)); + return new ColumnOrSuperColumn().setColumn(thriftifySubColumn(cell, newName)); } - private Column thriftifySubColumn(Cell cell) + private Column thriftifySubColumn(CFMetaData metadata, LegacyLayout.LegacyCell cell) { - assert !(cell instanceof CounterCell); + return thriftifySubColumn(cell, cell.name.encode(metadata)); + } - Column thrift_column = new Column(cell.name().toByteBuffer()).setValue(cell.value()).setTimestamp(cell.timestamp()); - if (cell instanceof ExpiringCell) - { - thrift_column.setTtl(((ExpiringCell) cell).getTimeToLive()); - } + private Column thriftifySubColumn(LegacyLayout.LegacyCell cell, ByteBuffer name) + { + assert !cell.isCounter(); + + Column thrift_column = new Column(name).setValue(cell.value).setTimestamp(cell.timestamp); + if (cell.isExpiring()) + thrift_column.setTtl(cell.ttl); return thrift_column; } - private List thriftifyColumnsAsColumns(Collection cells, long now) + private List thriftifyColumnsAsColumns(CFMetaData metadata, Iterator cells) { - List thriftColumns = new ArrayList(cells.size()); - for (Cell cell : cells) - { - if (!cell.isLive(now)) - continue; - - thriftColumns.add(thriftifySubColumn(cell)); - } + List thriftColumns = new ArrayList(); + while (cells.hasNext()) + thriftColumns.add(thriftifySubColumn(metadata, cells.next())); return thriftColumns; } - private CounterColumn thriftifySubCounter(Cell cell) + private CounterColumn thriftifySubCounter(CFMetaData metadata, LegacyLayout.LegacyCell cell) { - assert cell instanceof CounterCell; - return new CounterColumn(cell.name().toByteBuffer(), CounterContext.instance().total(cell.value())); + assert cell.isCounter(); + return new CounterColumn(cell.name.encode(metadata), CounterContext.instance().total(cell.value)); } - private List thriftifySuperColumns(Collection cells, - boolean reverseOrder, - long now, + private List thriftifySuperColumns(CFMetaData metadata, + Iterator cells, boolean subcolumnsOnly, - boolean isCounterCF) + boolean isCounterCF, + boolean reversed) { if (subcolumnsOnly) { - ArrayList thriftSuperColumns = new ArrayList(cells.size()); - for (Cell cell : cells) + ArrayList thriftSuperColumns = new ArrayList(); + while (cells.hasNext()) { - if (!cell.isLive(now)) - continue; - - thriftSuperColumns.add(thriftifyColumnWithName(cell, SuperColumns.subName(cell.name()))); + LegacyLayout.LegacyCell cell = cells.next(); + thriftSuperColumns.add(thriftifyColumnWithName(metadata, cell, cell.name.superColumnSubName())); } - if (reverseOrder) + // Generally, cells come reversed if the query is reverse. However, this is not the case within a super column because + // internally a super column is a map within a row and those are never returned reversed. + if (reversed) Collections.reverse(thriftSuperColumns); return thriftSuperColumns; } else { if (isCounterCF) - return thriftifyCounterSuperColumns(cells, reverseOrder, now); + return thriftifyCounterSuperColumns(metadata, cells, reversed); else - return thriftifySuperColumns(cells, reverseOrder, now); + return thriftifySuperColumns(cells, reversed); } } - private List thriftifySuperColumns(Collection cells, boolean reverseOrder, long now) + private List thriftifySuperColumns(Iterator cells, boolean reversed) { - ArrayList thriftSuperColumns = new ArrayList(cells.size()); + ArrayList thriftSuperColumns = new ArrayList(); SuperColumn current = null; - for (Cell cell : cells) + while (cells.hasNext()) { - if (!cell.isLive(now)) - continue; - - ByteBuffer scName = SuperColumns.scName(cell.name()); + LegacyLayout.LegacyCell cell = cells.next(); + ByteBuffer scName = cell.name.superColumnName(); if (current == null || !scName.equals(current.bufferForName())) { + // Generally, cells come reversed if the query is reverse. However, this is not the case within a super column because + // internally a super column is a map within a row and those are never returned reversed. + if (current != null && reversed) + Collections.reverse(current.columns); + current = new SuperColumn(scName, new ArrayList()); thriftSuperColumns.add(new ColumnOrSuperColumn().setSuper_column(current)); } - current.getColumns().add(thriftifySubColumn(cell).setName(SuperColumns.subName(cell.name()))); + current.getColumns().add(thriftifySubColumn(cell, cell.name.superColumnSubName())); } - if (reverseOrder) - Collections.reverse(thriftSuperColumns); + if (current != null && reversed) + Collections.reverse(current.columns); return thriftSuperColumns; } - private List thriftifyCounterSuperColumns(Collection cells, boolean reverseOrder, long now) + private List thriftifyCounterSuperColumns(CFMetaData metadata, Iterator cells, boolean reversed) { - ArrayList thriftSuperColumns = new ArrayList(cells.size()); + ArrayList thriftSuperColumns = new ArrayList(); CounterSuperColumn current = null; - for (Cell cell : cells) + while (cells.hasNext()) { - if (!cell.isLive(now)) - continue; - - ByteBuffer scName = SuperColumns.scName(cell.name()); + LegacyLayout.LegacyCell cell = cells.next(); + ByteBuffer scName = cell.name.superColumnName(); if (current == null || !scName.equals(current.bufferForName())) { + // Generally, cells come reversed if the query is reverse. However, this is not the case within a super column because + // internally a super column is a map within a row and those are never returned reversed. + if (current != null && reversed) + Collections.reverse(current.columns); + current = new CounterSuperColumn(scName, new ArrayList()); thriftSuperColumns.add(new ColumnOrSuperColumn().setCounter_super_column(current)); } - current.getColumns().add(thriftifySubCounter(cell).setName(SuperColumns.subName(cell.name()))); + current.getColumns().add(thriftifySubCounter(metadata, cell).setName(cell.name.superColumnSubName())); } - - if (reverseOrder) - Collections.reverse(thriftSuperColumns); - return thriftSuperColumns; } - private Map> getSlice(List commands, boolean subColumnsOnly, org.apache.cassandra.db.ConsistencyLevel consistency_level, ClientState cState) - throws org.apache.cassandra.exceptions.InvalidRequestException, UnavailableException, TimedOutException + private List thriftifyPartition(RowIterator partition, boolean subcolumnsOnly, boolean reversed, int cellLimit) { - Map columnFamilies = readColumnFamily(commands, consistency_level, cState); - Map> columnFamiliesMap = new HashMap>(); - for (ReadCommand command: commands) - { - ColumnFamily cf = columnFamilies.get(StorageService.getPartitioner().decorateKey(command.key)); - boolean reverseOrder = command instanceof SliceFromReadCommand && ((SliceFromReadCommand)command).filter.reversed; - List thriftifiedColumns = thriftifyColumnFamily(cf, subColumnsOnly, reverseOrder, command.timestamp); - columnFamiliesMap.put(command.key, thriftifiedColumns); - } - - return columnFamiliesMap; - } - - private List thriftifyColumnFamily(ColumnFamily cf, boolean subcolumnsOnly, boolean reverseOrder, long now) - { - if (cf == null || !cf.hasColumns()) + if (partition.isEmpty()) return EMPTY_COLUMNS; - if (cf.metadata().isSuper()) + Iterator cells = LegacyLayout.fromRowIterator(partition); + List result; + if (partition.metadata().isSuper()) { - boolean isCounterCF = cf.metadata().isCounter(); - return thriftifySuperColumns(cf.getSortedColumns(), reverseOrder, now, subcolumnsOnly, isCounterCF); + boolean isCounterCF = partition.metadata().isCounter(); + result = thriftifySuperColumns(partition.metadata(), cells, subcolumnsOnly, isCounterCF, reversed); } else { - return thriftifyColumns(cf.getSortedColumns(), reverseOrder, now); + result = thriftifyColumns(partition.metadata(), cells); + } + + // Thrift count cells, but internally we only count them at "row" boundaries, which means that if the limit stops in the middle + // of an internal row we'll include a few additional cells. So trim it here. + return result.size() > cellLimit + ? result.subList(0, cellLimit) + : result; + } + + private Map> getSlice(List> commands, boolean subColumnsOnly, int cellLimit, org.apache.cassandra.db.ConsistencyLevel consistency_level, ClientState cState) + throws org.apache.cassandra.exceptions.InvalidRequestException, UnavailableException, TimedOutException + { + try (PartitionIterator results = read(commands, consistency_level, cState)) + { + Map> columnFamiliesMap = new HashMap>(); + while (results.hasNext()) + { + try (RowIterator iter = results.next()) + { + List thriftifiedColumns = thriftifyPartition(iter, subColumnsOnly, iter.isReverseOrder(), cellLimit); + columnFamiliesMap.put(iter.partitionKey().getKey(), thriftifiedColumns); + } + } + return columnFamiliesMap; } } @@ -303,7 +294,8 @@ public class CassandraServer implements Cassandra.Iface ClientState cState = state(); String keyspace = cState.getKeyspace(); state().hasColumnFamilyAccess(keyspace, column_parent.column_family, Permission.SELECT); - return getSliceInternal(keyspace, key, column_parent, System.currentTimeMillis(), predicate, consistency_level, cState); + List result = getSliceInternal(keyspace, key, column_parent, FBUtilities.nowInSeconds(), predicate, consistency_level, cState); + return result == null ? Collections.emptyList() : result; } catch (RequestValidationException e) { @@ -318,13 +310,13 @@ public class CassandraServer implements Cassandra.Iface private List getSliceInternal(String keyspace, ByteBuffer key, ColumnParent column_parent, - long timestamp, + int nowInSec, SlicePredicate predicate, ConsistencyLevel consistency_level, ClientState cState) throws org.apache.cassandra.exceptions.InvalidRequestException, UnavailableException, TimedOutException { - return multigetSliceInternal(keyspace, Collections.singletonList(key), column_parent, timestamp, predicate, consistency_level, cState).get(key); + return multigetSliceInternal(keyspace, Collections.singletonList(key), column_parent, nowInSec, predicate, consistency_level, cState).get(key); } public Map> multiget_slice(List keys, ColumnParent column_parent, SlicePredicate predicate, ConsistencyLevel consistency_level) @@ -351,7 +343,7 @@ public class CassandraServer implements Cassandra.Iface ClientState cState = state(); String keyspace = cState.getKeyspace(); cState.hasColumnFamilyAccess(keyspace, column_parent.column_family, Permission.SELECT); - return multigetSliceInternal(keyspace, keys, column_parent, System.currentTimeMillis(), predicate, consistency_level, cState); + return multigetSliceInternal(keyspace, keys, column_parent, FBUtilities.nowInSeconds(), predicate, consistency_level, cState); } catch (RequestValidationException e) { @@ -363,55 +355,179 @@ public class CassandraServer implements Cassandra.Iface } } - private SliceQueryFilter toInternalFilter(CFMetaData metadata, ColumnParent parent, SliceRange range) + private ClusteringIndexFilter toInternalFilter(CFMetaData metadata, ColumnParent parent, SliceRange range) { - if (metadata.isSuper()) - { - CellNameType columnType = new SimpleDenseCellNameType(metadata.comparator.subtype(parent.isSetSuper_column() ? 1 : 0)); - Composite start = columnType.fromByteBuffer(range.start); - Composite finish = columnType.fromByteBuffer(range.finish); - SliceQueryFilter filter = new SliceQueryFilter(start, finish, range.reversed, range.count); - return SuperColumns.fromSCSliceFilter(metadata.comparator, parent.bufferForSuper_column(), filter); - } - - Composite start = metadata.comparator.fromByteBuffer(range.start); - Composite finish = metadata.comparator.fromByteBuffer(range.finish); - return new SliceQueryFilter(start, finish, range.reversed, range.count); + if (metadata.isSuper() && parent.isSetSuper_column()) + return new ClusteringIndexNamesFilter(FBUtilities.singleton(new SimpleClustering(parent.bufferForSuper_column()), metadata.comparator), range.reversed); + else + return new ClusteringIndexSliceFilter(makeSlices(metadata, range), range.reversed); } - private IDiskAtomFilter toInternalFilter(CFMetaData metadata, ColumnParent parent, SlicePredicate predicate) + private Slices makeSlices(CFMetaData metadata, SliceRange range) { - IDiskAtomFilter filter; + // Note that in thrift, the bounds are reversed if the query is reversed, but not internally. + ByteBuffer start = range.reversed ? range.finish : range.start; + ByteBuffer finish = range.reversed ? range.start : range.finish; + return Slices.with(metadata.comparator, Slice.make(LegacyLayout.decodeBound(metadata, start, true).bound, LegacyLayout.decodeBound(metadata, finish, false).bound)); + } - if (predicate.column_names != null) + private ClusteringIndexFilter toInternalFilter(CFMetaData metadata, ColumnParent parent, SlicePredicate predicate) + throws org.apache.cassandra.exceptions.InvalidRequestException + { + try { - if (metadata.isSuper()) + if (predicate.column_names != null) { - CellNameType columnType = new SimpleDenseCellNameType(metadata.comparator.subtype(parent.isSetSuper_column() ? 1 : 0)); - SortedSet s = new TreeSet<>(columnType); - for (ByteBuffer bb : predicate.column_names) - s.add(columnType.cellFromByteBuffer(bb)); - filter = SuperColumns.fromSCNamesFilter(metadata.comparator, parent.bufferForSuper_column(), new NamesQueryFilter(s)); + if (metadata.isSuper()) + { + if (parent.isSetSuper_column()) + { + return new ClusteringIndexNamesFilter(FBUtilities.singleton(new SimpleClustering(parent.bufferForSuper_column()), metadata.comparator), false); + } + else + { + NavigableSet clusterings = new TreeSet<>(metadata.comparator); + for (ByteBuffer bb : predicate.column_names) + clusterings.add(new SimpleClustering(bb)); + return new ClusteringIndexNamesFilter(clusterings, false); + } + } + else + { + NavigableSet clusterings = new TreeSet<>(metadata.comparator); + for (ByteBuffer bb : predicate.column_names) + { + LegacyLayout.LegacyCellName name = LegacyLayout.decodeCellName(metadata, parent.bufferForSuper_column(), bb); + clusterings.add(name.clustering); + } + return new ClusteringIndexNamesFilter(clusterings, false); + } } else { - SortedSet s = new TreeSet(metadata.comparator); - for (ByteBuffer bb : predicate.column_names) - s.add(metadata.comparator.cellFromByteBuffer(bb)); - filter = new NamesQueryFilter(s); + return toInternalFilter(metadata, parent, predicate.slice_range); } } - else + catch (UnknownColumnException e) { - filter = toInternalFilter(metadata, parent, predicate.slice_range); + throw new org.apache.cassandra.exceptions.InvalidRequestException(e.getMessage()); } - return filter; + } + + private ColumnFilter makeColumnFilter(CFMetaData metadata, ColumnParent parent, SliceRange range) + { + if (metadata.isSuper() && parent.isSetSuper_column()) + { + // We want a slice of the dynamic columns + ColumnFilter.Builder builder = ColumnFilter.selectionBuilder(); + ColumnDefinition def = metadata.compactValueColumn(); + ByteBuffer start = range.reversed ? range.finish : range.start; + ByteBuffer finish = range.reversed ? range.start : range.finish; + builder.slice(def, start.hasRemaining() ? CellPath.create(start) : CellPath.BOTTOM, finish.hasRemaining() ? CellPath.create(finish) : CellPath.TOP); + + // We also want to add any staticly defined column if it's within the range + AbstractType cmp = metadata.thriftColumnNameType(); + for (ColumnDefinition column : metadata.partitionColumns()) + { + if (CompactTables.isSuperColumnMapColumn(column)) + continue; + + ByteBuffer name = column.name.bytes; + if (cmp.compare(name, start) < 0 || cmp.compare(finish, name) > 0) + continue; + + builder.add(column); + } + return builder.build(); + } + return makeColumnFilter(metadata, makeSlices(metadata, range)); + } + + private ColumnFilter makeColumnFilter(CFMetaData metadata, Slices slices) + { + PartitionColumns columns = metadata.partitionColumns(); + if (metadata.isStaticCompactTable() && !columns.statics.isEmpty()) + { + PartitionColumns.Builder builder = PartitionColumns.builder(); + builder.addAll(columns.regulars); + // We only want to include the static columns that are selected by the slices + for (ColumnDefinition def : columns.statics) + { + if (slices.selects(new SimpleClustering(def.name.bytes))) + builder.add(def); + } + columns = builder.build(); + } + return ColumnFilter.selection(columns); + } + + private ColumnFilter makeColumnFilter(CFMetaData metadata, ColumnParent parent, SlicePredicate predicate) + throws org.apache.cassandra.exceptions.InvalidRequestException + { + try + { + if (predicate.column_names != null) + { + if (metadata.isSuper()) + { + if (parent.isSetSuper_column()) + { + ColumnFilter.Builder builder = ColumnFilter.selectionBuilder(); + ColumnDefinition dynamicDef = metadata.compactValueColumn(); + for (ByteBuffer bb : predicate.column_names) + { + ColumnDefinition staticDef = metadata.getColumnDefinition(bb); + if (staticDef == null) + builder.select(dynamicDef, CellPath.create(bb)); + else + builder.add(staticDef); + } + return builder.build(); + } + else + { + return ColumnFilter.all(metadata); + } + } + else + { + PartitionColumns.Builder builder = new PartitionColumns.Builder(); + for (ByteBuffer bb : predicate.column_names) + { + LegacyLayout.LegacyCellName name = LegacyLayout.decodeCellName(metadata, parent.bufferForSuper_column(), bb); + builder.add(name.column); + } + return ColumnFilter.selection(builder.build()); + } + } + else + { + return makeColumnFilter(metadata, parent, predicate.slice_range); + } + } + catch (UnknownColumnException e) + { + throw new org.apache.cassandra.exceptions.InvalidRequestException(e.getMessage()); + } + } + + private DataLimits getLimits(int partitionLimit, boolean countSuperColumns, SlicePredicate predicate) + { + int cellsPerPartition = predicate.slice_range == null ? Integer.MAX_VALUE : predicate.slice_range.count; + return getLimits(partitionLimit, countSuperColumns, cellsPerPartition); + } + + private DataLimits getLimits(int partitionLimit, boolean countSuperColumns, int perPartitionCount) + { + return countSuperColumns + ? DataLimits.superColumnCountingLimits(partitionLimit, perPartitionCount) + : DataLimits.thriftLimits(partitionLimit, perPartitionCount); } private Map> multigetSliceInternal(String keyspace, List keys, ColumnParent column_parent, - long timestamp, + int nowInSec, SlicePredicate predicate, ConsistencyLevel consistency_level, ClientState cState) @@ -424,18 +540,19 @@ public class CassandraServer implements Cassandra.Iface org.apache.cassandra.db.ConsistencyLevel consistencyLevel = ThriftConversion.fromThrift(consistency_level); consistencyLevel.validateForRead(keyspace); - List commands = new ArrayList(keys.size()); - IDiskAtomFilter filter = toInternalFilter(metadata, column_parent, predicate); + List> commands = new ArrayList<>(keys.size()); + ColumnFilter columnFilter = makeColumnFilter(metadata, column_parent, predicate); + ClusteringIndexFilter filter = toInternalFilter(metadata, column_parent, predicate); + DataLimits limits = getLimits(1, metadata.isSuper() && !column_parent.isSetSuper_column(), predicate); for (ByteBuffer key: keys) { ThriftValidation.validateKey(metadata, key); - // Note that we should not share a slice filter amongst the command, due to SliceQueryFilter not being immutable - // due to its columnCounter used by the lastCounted() method (also see SelectStatement.getSliceCommands) - commands.add(ReadCommand.create(keyspace, key, column_parent.getColumn_family(), timestamp, filter.cloneShallow())); + DecoratedKey dk = StorageService.getPartitioner().decorateKey(key); + commands.add(SinglePartitionReadCommand.create(true, metadata, nowInSec, columnFilter, RowFilter.NONE, limits, dk, filter)); } - return getSlice(commands, column_parent.isSetSuper_column(), consistencyLevel, cState); + return getSlice(commands, column_parent.isSetSuper_column(), limits.perPartitionCount(), consistencyLevel, cState); } public ColumnOrSuperColumn get(ByteBuffer key, ColumnPath column_path, ConsistencyLevel consistency_level) @@ -466,35 +583,58 @@ public class CassandraServer implements Cassandra.Iface ThriftValidation.validateKey(metadata, key); - IDiskAtomFilter filter; + ColumnFilter columns; + ClusteringIndexFilter filter; if (metadata.isSuper()) { - CellNameType columnType = new SimpleDenseCellNameType(metadata.comparator.subtype(column_path.column == null ? 0 : 1)); - SortedSet names = new TreeSet(columnType); - names.add(columnType.cellFromByteBuffer(column_path.column == null ? column_path.super_column : column_path.column)); - filter = SuperColumns.fromSCNamesFilter(metadata.comparator, column_path.column == null ? null : column_path.bufferForSuper_column(), new NamesQueryFilter(names)); + if (column_path.column == null) + { + // Selects a full super column + columns = ColumnFilter.all(metadata); + } + else + { + // Selects a single column within a super column + ColumnFilter.Builder builder = ColumnFilter.selectionBuilder(); + ColumnDefinition staticDef = metadata.getColumnDefinition(column_path.column); + ColumnDefinition dynamicDef = metadata.compactValueColumn(); + + if (staticDef != null) + builder.add(staticDef); + // Note that even if there is a staticDef, we still query the dynamicDef since we can't guarantee the static one hasn't + // been created after data has been inserted for that definition + builder.select(dynamicDef, CellPath.create(column_path.column)); + columns = builder.build(); + } + filter = new ClusteringIndexNamesFilter(FBUtilities.singleton(new SimpleClustering(column_path.super_column), metadata.comparator), + false); } else { - SortedSet names = new TreeSet(metadata.comparator); - names.add(metadata.comparator.cellFromByteBuffer(column_path.column)); - filter = new NamesQueryFilter(names); + LegacyLayout.LegacyCellName cellname = LegacyLayout.decodeCellName(metadata, column_path.super_column, column_path.column); + columns = ColumnFilter.selection(PartitionColumns.of(cellname.column)); + filter = new ClusteringIndexNamesFilter(FBUtilities.singleton(cellname.clustering, metadata.comparator), false); } long now = System.currentTimeMillis(); - ReadCommand command = ReadCommand.create(keyspace, key, column_path.column_family, now, filter); + DecoratedKey dk = StorageService.getPartitioner().decorateKey(key); + SinglePartitionReadCommand command = SinglePartitionReadCommand.create(true, metadata, FBUtilities.nowInSeconds(), columns, RowFilter.NONE, DataLimits.NONE, dk, filter); - Map cfamilies = readColumnFamily(Arrays.asList(command), consistencyLevel, cState); + try (RowIterator result = PartitionIterators.getOnlyElement(read(Arrays.>asList(command), consistencyLevel, cState), command)) + { + if (!result.hasNext()) + throw new NotFoundException(); - ColumnFamily cf = cfamilies.get(StorageService.getPartitioner().decorateKey(command.key)); - - if (cf == null) - throw new NotFoundException(); - List tcolumns = thriftifyColumnFamily(cf, metadata.isSuper() && column_path.column != null, false, now); - if (tcolumns.isEmpty()) - throw new NotFoundException(); - assert tcolumns.size() == 1; - return tcolumns.get(0); + List tcolumns = thriftifyPartition(result, metadata.isSuper() && column_path.column != null, result.isReverseOrder(), 1); + if (tcolumns.isEmpty()) + throw new NotFoundException(); + assert tcolumns.size() == 1; + return tcolumns.get(0); + } + } + catch (UnknownColumnException e) + { + throw new InvalidRequestException(e.getMessage()); } catch (RequestValidationException e) { @@ -529,10 +669,10 @@ public class CassandraServer implements Cassandra.Iface cState.hasColumnFamilyAccess(keyspace, column_parent.column_family, Permission.SELECT); Keyspace keyspaceName = Keyspace.open(keyspace); ColumnFamilyStore cfs = keyspaceName.getColumnFamilyStore(column_parent.column_family); - long timestamp = System.currentTimeMillis(); + int nowInSec = FBUtilities.nowInSeconds(); if (predicate.column_names != null) - return getSliceInternal(keyspace, key, column_parent, timestamp, predicate, consistency_level, cState).size(); + return getSliceInternal(keyspace, key, column_parent, nowInSec, predicate, consistency_level, cState).size(); int pageSize; // request by page if this is a large row @@ -551,16 +691,34 @@ public class CassandraServer implements Cassandra.Iface SliceRange sliceRange = predicate.slice_range == null ? new SliceRange(ByteBufferUtil.EMPTY_BYTE_BUFFER, ByteBufferUtil.EMPTY_BYTE_BUFFER, false, Integer.MAX_VALUE) : predicate.slice_range; - SliceQueryFilter filter = toInternalFilter(cfs.metadata, column_parent, sliceRange); - return QueryPagers.countPaged(keyspace, - column_parent.column_family, - key, + ColumnFilter columnFilter; + ClusteringIndexFilter filter; + if (cfs.metadata.isSuper() && !column_parent.isSetSuper_column()) + { + // If we count on a super column table without having set the super column name, we're in fact interested by the count of super columns + columnFilter = ColumnFilter.all(cfs.metadata); + filter = new ClusteringIndexSliceFilter(makeSlices(cfs.metadata, sliceRange), sliceRange.reversed); + } + else + { + columnFilter = makeColumnFilter(cfs.metadata, column_parent, sliceRange); + filter = toInternalFilter(cfs.metadata, column_parent, sliceRange); + } + + DataLimits limits = getLimits(1, cfs.metadata.isSuper() && !column_parent.isSetSuper_column(), predicate); + DecoratedKey dk = StorageService.getPartitioner().decorateKey(key); + + return QueryPagers.countPaged(cfs.metadata, + dk, + columnFilter, filter, + limits, ThriftConversion.fromThrift(consistency_level), cState, pageSize, - timestamp); + nowInSec, + true); } catch (IllegalArgumentException e) { @@ -612,7 +770,7 @@ public class CassandraServer implements Cassandra.Iface Map> columnFamiliesMap = multigetSliceInternal(keyspace, keys, column_parent, - System.currentTimeMillis(), + FBUtilities.nowInSeconds(), predicate, consistency_level, cState); @@ -642,25 +800,30 @@ public class CassandraServer implements Cassandra.Iface ThriftValidation.validateKey(metadata, key); ThriftValidation.validateColumnParent(metadata, column_parent); // SuperColumn field is usually optional, but not when we're inserting - if (metadata.cfType == ColumnFamilyType.Super && column_parent.super_column == null) + if (metadata.isSuper() && column_parent.super_column == null) { throw new org.apache.cassandra.exceptions.InvalidRequestException("missing mandatory super column name for super CF " + column_parent.column_family); } ThriftValidation.validateColumnNames(metadata, column_parent, Arrays.asList(column.name)); - ThriftValidation.validateColumnData(metadata, key, column_parent.super_column, column); + ThriftValidation.validateColumnData(metadata, column_parent.super_column, column); org.apache.cassandra.db.Mutation mutation; try { - CellName name = metadata.isSuper() - ? metadata.comparator.makeCellName(column_parent.super_column, column.name) - : metadata.comparator.cellFromByteBuffer(column.name); + LegacyLayout.LegacyCellName name = LegacyLayout.decodeCellName(metadata, column_parent.super_column, column.name); - ColumnFamily cf = ArrayBackedSortedColumns.factory.create(cState.getKeyspace(), column_parent.column_family); - cf.addColumn(name, column.value, column.timestamp, column.ttl); - mutation = new org.apache.cassandra.db.Mutation(cState.getKeyspace(), key, cf); + DecoratedKey dk = StorageService.getPartitioner().decorateKey(key); + PartitionUpdate update = new PartitionUpdate(metadata, dk, PartitionColumns.of(name.column), 1); + + Row.Writer writer = name.column.isStatic() ? update.staticWriter() : update.writer(); + name.clustering.writeTo(writer); + CellPath path = name.collectionElement == null ? null : CellPath.create(name.collectionElement); + writer.writeCell(name.column, false, column.value, SimpleLivenessInfo.forUpdate(column.timestamp, column.ttl, FBUtilities.nowInSeconds(), metadata), path); + writer.endOfRow(); + + mutation = new org.apache.cassandra.db.Mutation(update); } - catch (MarshalException e) + catch (MarshalException|UnknownColumnException e) { throw new org.apache.cassandra.exceptions.InvalidRequestException(e.getMessage()); } @@ -728,7 +891,7 @@ public class CassandraServer implements Cassandra.Iface CFMetaData metadata = ThriftValidation.validateColumnFamily(keyspace, column_family, false); ThriftValidation.validateKey(metadata, key); - if (metadata.cfType == ColumnFamilyType.Super) + if (metadata.isSuper()) throw new org.apache.cassandra.exceptions.InvalidRequestException("CAS does not support supercolumns"); Iterable names = Iterables.transform(updates, new Function() @@ -740,36 +903,36 @@ public class CassandraServer implements Cassandra.Iface }); ThriftValidation.validateColumnNames(metadata, new ColumnParent(column_family), names); for (Column column : updates) - ThriftValidation.validateColumnData(metadata, key, null, column); + ThriftValidation.validateColumnData(metadata, null, column); CFMetaData cfm = Schema.instance.getCFMetaData(cState.getKeyspace(), column_family); - ColumnFamily cfUpdates = ArrayBackedSortedColumns.factory.create(cfm); - for (Column column : updates) - cfUpdates.addColumn(cfm.comparator.cellFromByteBuffer(column.name), column.value, column.timestamp); - ColumnFamily cfExpected; - if (expected.isEmpty()) - { - cfExpected = null; - } - else - { - cfExpected = ArrayBackedSortedColumns.factory.create(cfm); - for (Column column : expected) - cfExpected.addColumn(cfm.comparator.cellFromByteBuffer(column.name), column.value, column.timestamp); - } + DecoratedKey dk = StorageService.getPartitioner().decorateKey(key); + int nowInSec = FBUtilities.nowInSeconds(); + + PartitionUpdate partitionUpdates = RowIterators.toUpdate(LegacyLayout.toRowIterator(metadata, dk, toLegacyCells(metadata, updates, nowInSec).iterator(), nowInSec)); + + FilteredPartition partitionExpected = null; + if (!expected.isEmpty()) + partitionExpected = FilteredPartition.create(LegacyLayout.toRowIterator(metadata, dk, toLegacyCells(metadata, expected, nowInSec).iterator(), nowInSec)); schedule(DatabaseDescriptor.getWriteRpcTimeout()); - ColumnFamily result = StorageProxy.cas(cState.getKeyspace(), - column_family, - key, - new ThriftCASRequest(cfExpected, cfUpdates), - ThriftConversion.fromThrift(serial_consistency_level), - ThriftConversion.fromThrift(commit_consistency_level), - cState); - return result == null - ? new CASResult(true) - : new CASResult(false).setCurrent_values(thriftifyColumnsAsColumns(result.getSortedColumns(), System.currentTimeMillis())); + try (RowIterator result = StorageProxy.cas(cState.getKeyspace(), + column_family, + dk, + new ThriftCASRequest(partitionExpected, partitionUpdates), + ThriftConversion.fromThrift(serial_consistency_level), + ThriftConversion.fromThrift(commit_consistency_level), + cState)) + { + return result == null + ? new CASResult(true) + : new CASResult(false).setCurrent_values(thriftifyColumnsAsColumns(metadata, LegacyLayout.fromRowIterator(result))); + } + } + catch (UnknownColumnException e) + { + throw new InvalidRequestException(e.getMessage()); } catch (RequestTimeoutException e) { @@ -789,18 +952,129 @@ public class CassandraServer implements Cassandra.Iface } } + private LegacyLayout.LegacyCell toLegacyCell(CFMetaData metadata, Column column, int nowInSec) throws UnknownColumnException + { + return toLegacyCell(metadata, null, column, nowInSec); + } + + private LegacyLayout.LegacyCell toLegacyCell(CFMetaData metadata, ByteBuffer superColumnName, Column column, int nowInSec) + throws UnknownColumnException + { + return column.ttl > 0 + ? LegacyLayout.LegacyCell.expiring(metadata, superColumnName, column.name, column.value, column.timestamp, column.ttl, nowInSec) + : LegacyLayout.LegacyCell.regular(metadata, superColumnName, column.name, column.value, column.timestamp); + } + + private LegacyLayout.LegacyCell toLegacyDeletion(CFMetaData metadata, ByteBuffer name, long timestamp, int nowInSec) + throws UnknownColumnException + { + return toLegacyDeletion(metadata, null, name, timestamp, nowInSec); + } + + private LegacyLayout.LegacyCell toLegacyDeletion(CFMetaData metadata, ByteBuffer superColumnName, ByteBuffer name, long timestamp, int nowInSec) + throws UnknownColumnException + { + return LegacyLayout.LegacyCell.tombstone(metadata, superColumnName, name, timestamp, nowInSec); + } + + private LegacyLayout.LegacyCell toCounterLegacyCell(CFMetaData metadata, CounterColumn column) + throws UnknownColumnException + { + return toCounterLegacyCell(metadata, null, column); + } + + private LegacyLayout.LegacyCell toCounterLegacyCell(CFMetaData metadata, ByteBuffer superColumnName, CounterColumn column) + throws UnknownColumnException + { + return LegacyLayout.LegacyCell.counter(metadata, superColumnName, column.name, column.value); + } + + private void sortAndMerge(CFMetaData metadata, List cells, int nowInSec) + { + Collections.sort(cells, LegacyLayout.legacyCellComparator(metadata)); + + // After sorting, if we have multiple cells for the same "cellname", we want to merge those together. + Comparator comparator = LegacyLayout.legacyCellNameComparator(metadata, false); + + int previous = 0; // The last element that was set + for (int current = 1; current < cells.size(); current++) + { + LegacyLayout.LegacyCell pc = cells.get(previous); + LegacyLayout.LegacyCell cc = cells.get(current); + + // There is really only 2 possible comparison: < 0 or == 0 since we've sorted already + int cmp = comparator.compare(pc.name, cc.name); + if (cmp == 0) + { + // current and previous are the same cell. Merge current into previous + // (and so previous + 1 will be "free"). + Conflicts.Resolution res; + if (metadata.isCounter()) + { + res = Conflicts.resolveCounter(pc.timestamp, pc.isLive(nowInSec), pc.value, + cc.timestamp, cc.isLive(nowInSec), cc.value); + + } + else + { + res = Conflicts.resolveRegular(pc.timestamp, pc.isLive(nowInSec), pc.localDeletionTime, pc.value, + cc.timestamp, cc.isLive(nowInSec), cc.localDeletionTime, cc.value); + } + + switch (res) + { + case LEFT_WINS: + // The previous cell wins, we'll just ignore current + break; + case RIGHT_WINS: + cells.set(previous, cc); + break; + case MERGE: + assert metadata.isCounter(); + ByteBuffer merged = Conflicts.mergeCounterValues(pc.value, cc.value); + cells.set(previous, LegacyLayout.LegacyCell.counter(pc.name, merged)); + break; + } + } + else + { + // cell.get(previous) < cells.get(current), so move current just after previous if needs be + ++previous; + if (previous != current) + cells.set(previous, cc); + } + } + + // The last element we want is previous, so trim anything after that + for (int i = cells.size() - 1; i > previous; i--) + cells.remove(i); + } + + private List toLegacyCells(CFMetaData metadata, List columns, int nowInSec) + throws UnknownColumnException + { + List cells = new ArrayList<>(columns.size()); + for (Column column : columns) + cells.add(toLegacyCell(metadata, column, nowInSec)); + + sortAndMerge(metadata, cells, nowInSec); + return cells; + } + private List createMutationList(ConsistencyLevel consistency_level, Map>> mutation_map, boolean allowCounterMutations) - throws RequestValidationException + throws RequestValidationException, InvalidRequestException { List mutations = new ArrayList<>(); ThriftClientState cState = state(); String keyspace = cState.getKeyspace(); + int nowInSec = FBUtilities.nowInSeconds(); for (Map.Entry>> mutationEntry: mutation_map.entrySet()) { ByteBuffer key = mutationEntry.getKey(); + DecoratedKey dk = StorageService.getPartitioner().decorateKey(key); // We need to separate mutation for standard cf and counter cf (that will be encapsulated in a // CounterMutation) because it doesn't follow the same code path @@ -811,38 +1085,46 @@ public class CassandraServer implements Cassandra.Iface for (Map.Entry> columnFamilyMutations : columnFamilyToMutations.entrySet()) { String cfName = columnFamilyMutations.getKey(); + List muts = columnFamilyMutations.getValue(); cState.hasColumnFamilyAccess(keyspace, cfName, Permission.MODIFY); CFMetaData metadata = ThriftValidation.validateColumnFamily(keyspace, cfName); ThriftValidation.validateKey(metadata, key); + if (metadata.isCounter()) + ThriftConversion.fromThrift(consistency_level).validateCounterForWrite(metadata); + + DeletionInfo delInfo = DeletionInfo.live(); + List cells = new ArrayList<>(); + for (Mutation m : muts) + { + ThriftValidation.validateMutation(metadata, m); + + if (m.deletion != null) + { + deleteColumnOrSuperColumn(delInfo, cells, metadata, m.deletion, nowInSec); + } + if (m.column_or_supercolumn != null) + { + addColumnOrSuperColumn(cells, metadata, m.column_or_supercolumn, nowInSec); + } + } + + sortAndMerge(metadata, cells, nowInSec); + PartitionUpdate update = UnfilteredRowIterators.toUpdate(LegacyLayout.toUnfilteredRowIterator(metadata, dk, delInfo, cells.iterator())); org.apache.cassandra.db.Mutation mutation; if (metadata.isCounter()) { - ThriftConversion.fromThrift(consistency_level).validateCounterForWrite(metadata); - counterMutation = counterMutation == null ? new org.apache.cassandra.db.Mutation(keyspace, key) : counterMutation; + counterMutation = counterMutation == null ? new org.apache.cassandra.db.Mutation(keyspace, dk) : counterMutation; mutation = counterMutation; } else { - standardMutation = standardMutation == null ? new org.apache.cassandra.db.Mutation(keyspace, key) : standardMutation; + standardMutation = standardMutation == null ? new org.apache.cassandra.db.Mutation(keyspace, dk) : standardMutation; mutation = standardMutation; } - - for (Mutation m : columnFamilyMutations.getValue()) - { - ThriftValidation.validateMutation(metadata, key, m); - - if (m.deletion != null) - { - deleteColumnOrSuperColumn(mutation, metadata, m.deletion); - } - if (m.column_or_supercolumn != null) - { - addColumnOrSuperColumn(mutation, metadata, m.column_or_supercolumn); - } - } + mutation.add(update); } if (standardMutation != null && !standardMutation.isEmpty()) mutations.add(standardMutation); @@ -859,70 +1141,91 @@ public class CassandraServer implements Cassandra.Iface return mutations; } - private void addColumnOrSuperColumn(org.apache.cassandra.db.Mutation mutation, CFMetaData cfm, ColumnOrSuperColumn cosc) + private void addColumnOrSuperColumn(List cells, CFMetaData cfm, ColumnOrSuperColumn cosc, int nowInSec) + throws InvalidRequestException { - if (cosc.super_column != null) + try { - for (Column column : cosc.super_column.columns) + if (cosc.super_column != null) { - mutation.add(cfm.cfName, cfm.comparator.makeCellName(cosc.super_column.name, column.name), column.value, column.timestamp, column.ttl); + for (Column column : cosc.super_column.columns) + cells.add(toLegacyCell(cfm, cosc.super_column.name, column, nowInSec)); + } + else if (cosc.column != null) + { + cells.add(toLegacyCell(cfm, cosc.column, nowInSec)); + } + else if (cosc.counter_super_column != null) + { + for (CounterColumn column : cosc.counter_super_column.columns) + cells.add(toCounterLegacyCell(cfm, cosc.counter_super_column.name, column)); + } + else // cosc.counter_column != null + { + cells.add(toCounterLegacyCell(cfm, cosc.counter_column)); } } - else if (cosc.column != null) + catch (UnknownColumnException e) { - mutation.add(cfm.cfName, cfm.comparator.cellFromByteBuffer(cosc.column.name), cosc.column.value, cosc.column.timestamp, cosc.column.ttl); - } - else if (cosc.counter_super_column != null) - { - for (CounterColumn column : cosc.counter_super_column.columns) - { - mutation.addCounter(cfm.cfName, cfm.comparator.makeCellName(cosc.counter_super_column.name, column.name), column.value); - } - } - else // cosc.counter_column != null - { - mutation.addCounter(cfm.cfName, cfm.comparator.cellFromByteBuffer(cosc.counter_column.name), cosc.counter_column.value); + throw new InvalidRequestException(e.getMessage()); } } - private void deleteColumnOrSuperColumn(org.apache.cassandra.db.Mutation mutation, CFMetaData cfm, Deletion del) + private void addRange(CFMetaData cfm, DeletionInfo delInfo, Slice.Bound start, Slice.Bound end, long timestamp, int nowInSec) + { + delInfo.add(new RangeTombstone(Slice.make(start, end), new SimpleDeletionTime(timestamp, nowInSec)), cfm.comparator); + } + + private void deleteColumnOrSuperColumn(DeletionInfo delInfo, List cells, CFMetaData cfm, Deletion del, int nowInSec) + throws InvalidRequestException { if (del.predicate != null && del.predicate.column_names != null) { for (ByteBuffer c : del.predicate.column_names) { - if (del.super_column == null && cfm.isSuper()) - mutation.deleteRange(cfm.cfName, SuperColumns.startOf(c), SuperColumns.endOf(c), del.timestamp); - else if (del.super_column != null) - mutation.delete(cfm.cfName, cfm.comparator.makeCellName(del.super_column, c), del.timestamp); - else - mutation.delete(cfm.cfName, cfm.comparator.cellFromByteBuffer(c), del.timestamp); + try + { + if (del.super_column == null && cfm.isSuper()) + addRange(cfm, delInfo, Slice.Bound.inclusiveStartOf(c), Slice.Bound.inclusiveEndOf(c), del.timestamp, nowInSec); + else if (del.super_column != null) + cells.add(toLegacyDeletion(cfm, del.super_column, c, del.timestamp, nowInSec)); + else + cells.add(toLegacyDeletion(cfm, c, del.timestamp, nowInSec)); + } + catch (UnknownColumnException e) + { + throw new InvalidRequestException(e.getMessage()); + } } } else if (del.predicate != null && del.predicate.slice_range != null) { - if (del.super_column == null && cfm.isSuper()) - mutation.deleteRange(cfm.cfName, - SuperColumns.startOf(del.predicate.getSlice_range().start), - SuperColumns.endOf(del.predicate.getSlice_range().finish), - del.timestamp); - else if (del.super_column != null) - mutation.deleteRange(cfm.cfName, - cfm.comparator.makeCellName(del.super_column, del.predicate.getSlice_range().start), - cfm.comparator.makeCellName(del.super_column, del.predicate.getSlice_range().finish), - del.timestamp); + if (del.super_column == null) + { + addRange(cfm, + delInfo, + LegacyLayout.decodeBound(cfm, del.predicate.getSlice_range().start, true).bound, + LegacyLayout.decodeBound(cfm, del.predicate.getSlice_range().finish, false).bound, + del.timestamp, + nowInSec); + } else - mutation.deleteRange(cfm.cfName, - cfm.comparator.fromByteBuffer(del.predicate.getSlice_range().start), - cfm.comparator.fromByteBuffer(del.predicate.getSlice_range().finish), - del.timestamp); + { + // Since we use a map for subcolumns, we would need range tombstone for collections to support this. + // And while we may want those some day, this require a bit of additional work. And since super columns + // are basically deprecated since a long time, and range tombstone on them has been only very recently + // added so that no thrift driver actually supports it to the best of my knowledge, it's likely ok to + // discontinue support for this. If it turns out that this is blocking the update of someone, we can + // decide then if we want to tackle the addition of range tombstone for collections then. + throw new InvalidRequestException("Cannot delete a range of subcolumns in a super column"); + } } else { if (del.super_column != null) - mutation.deleteRange(cfm.cfName, SuperColumns.startOf(del.super_column), SuperColumns.endOf(del.super_column), del.timestamp); + addRange(cfm, delInfo, Slice.Bound.inclusiveStartOf(del.super_column), Slice.Bound.inclusiveEndOf(del.super_column), del.timestamp, nowInSec); else - mutation.delete(cfm.cfName, del.timestamp); + delInfo.add(new SimpleDeletionTime(del.timestamp, nowInSec)); } } @@ -1005,15 +1308,41 @@ public class CassandraServer implements Cassandra.Iface if (isCommutativeOp) ThriftConversion.fromThrift(consistency_level).validateCounterForWrite(metadata); - org.apache.cassandra.db.Mutation mutation = new org.apache.cassandra.db.Mutation(keyspace, key); + DecoratedKey dk = StorageService.getPartitioner().decorateKey(key); + + int nowInSec = FBUtilities.nowInSeconds(); + PartitionUpdate update; if (column_path.super_column == null && column_path.column == null) - mutation.delete(column_path.column_family, timestamp); - else if (column_path.super_column == null) - mutation.delete(column_path.column_family, metadata.comparator.cellFromByteBuffer(column_path.column), timestamp); - else if (column_path.column == null) - mutation.deleteRange(column_path.column_family, SuperColumns.startOf(column_path.super_column), SuperColumns.endOf(column_path.super_column), timestamp); + { + update = PartitionUpdate.fullPartitionDelete(metadata, dk, timestamp, nowInSec); + } + else if (column_path.super_column != null && column_path.column == null) + { + update = new PartitionUpdate(metadata, dk, PartitionColumns.NONE, 1); + Row.Writer writer = update.writer(); + writer.writeClusteringValue(column_path.super_column); + writer.writeRowDeletion(new SimpleDeletionTime(timestamp, nowInSec)); + writer.endOfRow(); + } else - mutation.delete(column_path.column_family, metadata.comparator.makeCellName(column_path.super_column, column_path.column), timestamp); + { + try + { + LegacyLayout.LegacyCellName name = LegacyLayout.decodeCellName(metadata, column_path.super_column, column_path.column); + update = new PartitionUpdate(metadata, dk, PartitionColumns.of(name.column), 1); + Row.Writer writer = name.column.isStatic() ? update.staticWriter() : update.writer(); + name.clustering.writeTo(writer); + CellPath path = name.collectionElement == null ? null : CellPath.create(name.collectionElement); + writer.writeCell(name.column, false, ByteBufferUtil.EMPTY_BYTE_BUFFER, SimpleLivenessInfo.forDeletion(timestamp, nowInSec), path); + writer.endOfRow(); + } + catch (UnknownColumnException e) + { + throw new org.apache.cassandra.exceptions.InvalidRequestException(e.getMessage()); + } + } + + org.apache.cassandra.db.Mutation mutation = new org.apache.cassandra.db.Mutation(update); if (isCommutativeOp) doInsert(consistency_level, Arrays.asList(new CounterMutation(mutation, ThriftConversion.fromThrift(consistency_level)))); @@ -1138,10 +1467,8 @@ public class CassandraServer implements Cassandra.Iface org.apache.cassandra.db.ConsistencyLevel consistencyLevel = ThriftConversion.fromThrift(consistency_level); consistencyLevel.validateForRead(keyspace); - List rows = null; - IPartitioner p = StorageService.getPartitioner(); - AbstractBounds bounds; + AbstractBounds bounds; if (range.start_key == null) { Token.TokenFactory tokenFactory = p.getTokenFactory(); @@ -1151,32 +1478,36 @@ public class CassandraServer implements Cassandra.Iface } else { - RowPosition end = range.end_key == null + PartitionPosition end = range.end_key == null ? p.getTokenFactory().fromString(range.end_token).maxKeyBound() - : RowPosition.ForKey.get(range.end_key, p); - bounds = new Bounds(RowPosition.ForKey.get(range.start_key, p), end); + : PartitionPosition.ForKey.get(range.end_key, p); + bounds = new Bounds(PartitionPosition.ForKey.get(range.start_key, p), end); } - long now = System.currentTimeMillis(); + int nowInSec = FBUtilities.nowInSeconds(); schedule(DatabaseDescriptor.getRangeRpcTimeout()); try { - IDiskAtomFilter filter = ThriftValidation.asIFilter(predicate, metadata, column_parent.super_column); - rows = StorageProxy.getRangeSlice(new RangeSliceCommand(keyspace, - column_parent.column_family, - now, - filter, - bounds, - ThriftConversion.indexExpressionsFromThrift(range.row_filter), - range.count), - consistencyLevel); + ColumnFilter columns = makeColumnFilter(metadata, column_parent, predicate); + ClusteringIndexFilter filter = toInternalFilter(metadata, column_parent, predicate); + DataLimits limits = getLimits(range.count, metadata.isSuper() && !column_parent.isSetSuper_column(), predicate); + PartitionRangeReadCommand cmd = new PartitionRangeReadCommand(false, + true, + metadata, + nowInSec, + columns, + ThriftConversion.rowFilterFromThrift(metadata, range.row_filter), + limits, + new DataRange(bounds, filter)); + try (PartitionIterator results = StorageProxy.getRangeSlice(cmd, consistencyLevel)) + { + assert results != null; + return thriftifyKeySlices(results, column_parent, limits.perPartitionCount()); + } } finally { release(); } - assert rows != null; - - return thriftifyKeySlices(rows, column_parent, predicate, now); } catch (RequestValidationException e) { @@ -1221,10 +1552,8 @@ public class CassandraServer implements Cassandra.Iface org.apache.cassandra.db.ConsistencyLevel consistencyLevel = ThriftConversion.fromThrift(consistency_level); consistencyLevel.validateForRead(keyspace); - SlicePredicate predicate = new SlicePredicate().setSlice_range(new SliceRange(start_column, ByteBufferUtil.EMPTY_BYTE_BUFFER, false, -1)); - IPartitioner p = StorageService.getPartitioner(); - AbstractBounds bounds; + AbstractBounds bounds; if (range.start_key == null) { // (token, key) is unsupported, assume (token, token) @@ -1235,30 +1564,45 @@ public class CassandraServer implements Cassandra.Iface } else { - RowPosition end = range.end_key == null + PartitionPosition end = range.end_key == null ? p.getTokenFactory().fromString(range.end_token).maxKeyBound() - : RowPosition.ForKey.get(range.end_key, p); - bounds = new Bounds(RowPosition.ForKey.get(range.start_key, p), end); + : PartitionPosition.ForKey.get(range.end_key, p); + bounds = new Bounds(PartitionPosition.ForKey.get(range.start_key, p), end); } if (range.row_filter != null && !range.row_filter.isEmpty()) throw new InvalidRequestException("Cross-row paging is not supported along with index clauses"); - List rows; - long now = System.currentTimeMillis(); + int nowInSec = FBUtilities.nowInSeconds(); schedule(DatabaseDescriptor.getRangeRpcTimeout()); try { - IDiskAtomFilter filter = ThriftValidation.asIFilter(predicate, metadata, null); - rows = StorageProxy.getRangeSlice(new RangeSliceCommand(keyspace, column_family, now, filter, bounds, null, range.count, true, true), consistencyLevel); + ClusteringIndexFilter filter = new ClusteringIndexSliceFilter(Slices.ALL, false); + DataLimits limits = getLimits(range.count, true, Integer.MAX_VALUE); + Clustering pageFrom = metadata.isSuper() + ? new SimpleClustering(start_column) + : LegacyLayout.decodeCellName(metadata, start_column).clustering; + PartitionRangeReadCommand cmd = new PartitionRangeReadCommand(false, + true, + metadata, + nowInSec, + ColumnFilter.all(metadata), + RowFilter.NONE, + limits, + new DataRange(bounds, filter).forPaging(bounds, metadata.comparator, pageFrom, true)); + try (PartitionIterator results = StorageProxy.getRangeSlice(cmd, consistencyLevel)) + { + return thriftifyKeySlices(results, new ColumnParent(column_family), limits.perPartitionCount()); + } + } + catch (UnknownColumnException e) + { + throw new InvalidRequestException(e.getMessage()); } finally { release(); } - assert rows != null; - - return thriftifyKeySlices(rows, new ColumnParent(column_family), predicate, now); } catch (RequestValidationException e) { @@ -1274,17 +1618,22 @@ public class CassandraServer implements Cassandra.Iface } } - private List thriftifyKeySlices(List rows, ColumnParent column_parent, SlicePredicate predicate, long now) + private List thriftifyKeySlices(PartitionIterator results, ColumnParent column_parent, int cellLimit) { - List keySlices = new ArrayList(rows.size()); - boolean reversed = predicate.slice_range != null && predicate.slice_range.reversed; - for (Row row : rows) + try (PartitionIterator iter = results) { - List thriftifiedColumns = thriftifyColumnFamily(row.cf, column_parent.super_column != null, reversed, now); - keySlices.add(new KeySlice(row.key.getKey(), thriftifiedColumns)); - } + List keySlices = new ArrayList(); + while (iter.hasNext()) + { + try (RowIterator partition = iter.next()) + { + List thriftifiedColumns = thriftifyPartition(partition, column_parent.super_column != null, partition.isReverseOrder(), cellLimit); + keySlices.add(new KeySlice(partition.partitionKey().getKey(), thriftifiedColumns)); + } + } - return keySlices; + return keySlices; + } } public List get_indexed_slices(ColumnParent column_parent, IndexClause index_clause, SlicePredicate column_predicate, ConsistencyLevel consistency_level) @@ -1316,21 +1665,25 @@ public class CassandraServer implements Cassandra.Iface consistencyLevel.validateForRead(keyspace); IPartitioner p = StorageService.getPartitioner(); - AbstractBounds bounds = new Bounds(RowPosition.ForKey.get(index_clause.start_key, p), + AbstractBounds bounds = new Bounds(PartitionPosition.ForKey.get(index_clause.start_key, p), p.getMinimumToken().minKeyBound()); - IDiskAtomFilter filter = ThriftValidation.asIFilter(column_predicate, metadata, column_parent.super_column); - long now = System.currentTimeMillis(); - RangeSliceCommand command = new RangeSliceCommand(keyspace, - column_parent.column_family, - now, - filter, - bounds, - ThriftConversion.indexExpressionsFromThrift(index_clause.expressions), - index_clause.count); - - List rows = StorageProxy.getRangeSlice(command, consistencyLevel); - return thriftifyKeySlices(rows, column_parent, column_predicate, now); + int nowInSec = FBUtilities.nowInSeconds(); + ColumnFilter columns = makeColumnFilter(metadata, column_parent, column_predicate); + ClusteringIndexFilter filter = toInternalFilter(metadata, column_parent, column_predicate); + DataLimits limits = getLimits(index_clause.count, metadata.isSuper() && !column_parent.isSetSuper_column(), column_predicate); + PartitionRangeReadCommand cmd = new PartitionRangeReadCommand(false, + true, + metadata, + nowInSec, + columns, + ThriftConversion.rowFilterFromThrift(metadata, index_clause.expressions), + limits, + new DataRange(bounds, filter)); + try (PartitionIterator results = StorageProxy.getRangeSlice(cmd, consistencyLevel)) + { + return thriftifyKeySlices(results, column_parent, limits.perPartitionCount()); + } } catch (RequestValidationException e) { @@ -1751,24 +2104,34 @@ public class CassandraServer implements Cassandra.Iface ThriftConversion.fromThrift(consistency_level).validateCounterForWrite(metadata); ThriftValidation.validateColumnParent(metadata, column_parent); // SuperColumn field is usually optional, but not when we're adding - if (metadata.cfType == ColumnFamilyType.Super && column_parent.super_column == null) + if (metadata.isSuper() && column_parent.super_column == null) throw new InvalidRequestException("missing mandatory super column name for super CF " + column_parent.column_family); ThriftValidation.validateColumnNames(metadata, column_parent, Arrays.asList(column.name)); - org.apache.cassandra.db.Mutation mutation = new org.apache.cassandra.db.Mutation(keyspace, key); try { - if (metadata.isSuper()) - mutation.addCounter(column_parent.column_family, metadata.comparator.makeCellName(column_parent.super_column, column.name), column.value); - else - mutation.addCounter(column_parent.column_family, metadata.comparator.cellFromByteBuffer(column.name), column.value); + LegacyLayout.LegacyCellName name = LegacyLayout.decodeCellName(metadata, column_parent.super_column, column.name); + DecoratedKey dk = StorageService.getPartitioner().decorateKey(key); + PartitionUpdate update = new PartitionUpdate(metadata, dk, PartitionColumns.of(name.column), 1); + + Row.Writer writer = name.column.isStatic() ? update.staticWriter() : update.writer(); + name.clustering.writeTo(writer); + CellPath path = name.collectionElement == null ? null : CellPath.create(name.collectionElement); + + // See UpdateParameters.addCounter() for more details on this + ByteBuffer value = CounterContext.instance().createLocal(column.value); + + writer.writeCell(name.column, true, value, SimpleLivenessInfo.forUpdate(FBUtilities.timestampMicros(), LivenessInfo.NO_TTL, FBUtilities.nowInSeconds(), metadata), path); + writer.endOfRow(); + + org.apache.cassandra.db.Mutation mutation = new org.apache.cassandra.db.Mutation(update); + doInsert(consistency_level, Arrays.asList(new CounterMutation(mutation, ThriftConversion.fromThrift(consistency_level)))); } - catch (MarshalException e) + catch (MarshalException|UnknownColumnException e) { throw new InvalidRequestException(e.getMessage()); } - doInsert(consistency_level, Arrays.asList(new CounterMutation(mutation, ThriftConversion.fromThrift(consistency_level)))); } catch (RequestValidationException e) { @@ -1797,7 +2160,7 @@ public class CassandraServer implements Cassandra.Iface try { - internal_remove(key, path, System.currentTimeMillis(), consistency_level, true); + internal_remove(key, path, FBUtilities.timestampMicros(), consistency_level, true); } catch (RequestValidationException e) { @@ -1869,7 +2232,7 @@ public class CassandraServer implements Cassandra.Iface public CqlResult execute_cql_query(ByteBuffer query, Compression compression) throws TException { - throw new InvalidRequestException("CQL2 has been removed in Cassandra 2.2. Please use CQL3 instead"); + throw new InvalidRequestException("CQL2 has been removed in Cassandra 3.0. Please use CQL3 instead"); } public CqlResult execute_cql3_query(ByteBuffer query, Compression compression, ConsistencyLevel cLevel) throws TException @@ -1890,7 +2253,8 @@ public class CassandraServer implements Cassandra.Iface ThriftClientState cState = state(); return ClientState.getCQLQueryHandler().process(queryString, cState.getQueryState(), - QueryOptions.fromProtocolV2(ThriftConversion.fromThrift(cLevel), Collections.emptyList()), + QueryOptions.fromProtocolV2(ThriftConversion.fromThrift(cLevel), + Collections.emptyList()), null).toThriftResult(); } catch (RequestExecutionException e) @@ -1909,7 +2273,7 @@ public class CassandraServer implements Cassandra.Iface public CqlPreparedResult prepare_cql_query(ByteBuffer query, Compression compression) throws TException { - throw new InvalidRequestException("CQL2 has been removed in Cassandra 2.2. Please use CQL3 instead"); + throw new InvalidRequestException("CQL2 has been removed in Cassandra 3.0. Please use CQL3 instead"); } public CqlPreparedResult prepare_cql3_query(ByteBuffer query, Compression compression) throws TException @@ -1922,9 +2286,7 @@ public class CassandraServer implements Cassandra.Iface try { cState.validateLogin(); - return ClientState.getCQLQueryHandler().prepare(queryString, - cState.getQueryState(), - null).toThriftPreparedResult(); + return ClientState.getCQLQueryHandler().prepare(queryString, cState.getQueryState(), null).toThriftPreparedResult(); } catch (RequestValidationException e) { @@ -1934,7 +2296,7 @@ public class CassandraServer implements Cassandra.Iface public CqlResult execute_prepared_cql_query(int itemId, List bindVariables) throws TException { - throw new InvalidRequestException("CQL2 has been removed in Cassandra 2.2. Please use CQL3 instead"); + throw new InvalidRequestException("CQL2 has been removed in Cassandra 3.0. Please use CQL3 instead"); } public CqlResult execute_prepared_cql3_query(int itemId, List bindVariables, ConsistencyLevel cLevel) throws TException @@ -2003,33 +2365,41 @@ public class CassandraServer implements Cassandra.Iface String keyspace = cState.getKeyspace(); state().hasColumnFamilyAccess(keyspace, request.getColumn_parent().column_family, Permission.SELECT); CFMetaData metadata = ThriftValidation.validateColumnFamily(keyspace, request.getColumn_parent().column_family); - if (metadata.cfType == ColumnFamilyType.Super) + if (metadata.isSuper()) throw new org.apache.cassandra.exceptions.InvalidRequestException("get_multi_slice does not support super columns"); ThriftValidation.validateColumnParent(metadata, request.getColumn_parent()); org.apache.cassandra.db.ConsistencyLevel consistencyLevel = ThriftConversion.fromThrift(request.getConsistency_level()); consistencyLevel.validateForRead(keyspace); - List commands = new ArrayList<>(1); - ColumnSlice[] slices = new ColumnSlice[request.getColumn_slices().size()]; + + Slices.Builder builder = new Slices.Builder(metadata.comparator, request.getColumn_slices().size()); for (int i = 0 ; i < request.getColumn_slices().size() ; i++) { fixOptionalSliceParameters(request.getColumn_slices().get(i)); - Composite start = metadata.comparator.fromByteBuffer(request.getColumn_slices().get(i).start); - Composite finish = metadata.comparator.fromByteBuffer(request.getColumn_slices().get(i).finish); - if (!start.isEmpty() && !finish.isEmpty()) - { - int compare = metadata.comparator.compare(start, finish); - if (!request.reversed && compare > 0) - throw new InvalidRequestException(String.format("Column slice at index %d had start greater than finish", i)); - else if (request.reversed && compare < 0) - throw new InvalidRequestException(String.format("Reversed column slice at index %d had start less than finish", i)); - } - slices[i] = new ColumnSlice(start, finish); + Slice.Bound start = LegacyLayout.decodeBound(metadata, request.getColumn_slices().get(i).start, true).bound; + Slice.Bound finish = LegacyLayout.decodeBound(metadata, request.getColumn_slices().get(i).finish, false).bound; + + int compare = metadata.comparator.compare(start, finish); + if (!request.reversed && compare > 0) + throw new InvalidRequestException(String.format("Column slice at index %d had start greater than finish", i)); + else if (request.reversed && compare < 0) + throw new InvalidRequestException(String.format("Reversed column slice at index %d had start less than finish", i)); + + builder.add(request.reversed ? Slice.make(finish, start) : Slice.make(start, finish)); } - ColumnSlice[] deoverlapped = ColumnSlice.deoverlapSlices(slices, request.reversed ? metadata.comparator.reverseComparator() : metadata.comparator); - SliceQueryFilter filter = new SliceQueryFilter(deoverlapped, request.reversed, request.count); + + Slices slices = builder.build(); + ColumnFilter columns = makeColumnFilter(metadata, slices); + ClusteringIndexSliceFilter filter = new ClusteringIndexSliceFilter(slices, request.reversed); + DataLimits limits = getLimits(1, false, request.count); + ThriftValidation.validateKey(metadata, request.key); - commands.add(ReadCommand.create(keyspace, request.key, request.column_parent.getColumn_family(), System.currentTimeMillis(), filter)); - return getSlice(commands, request.column_parent.isSetSuper_column(), consistencyLevel, cState).entrySet().iterator().next().getValue(); + DecoratedKey dk = StorageService.getPartitioner().decorateKey(request.key); + SinglePartitionReadCommand cmd = SinglePartitionReadCommand.create(true, metadata, FBUtilities.nowInSeconds(), columns, RowFilter.NONE, limits, dk, filter); + return getSlice(Collections.>singletonList(cmd), + false, + limits.perPartitionCount(), + consistencyLevel, + cState).entrySet().iterator().next().getValue(); } catch (RequestValidationException e) { @@ -2053,7 +2423,7 @@ public class CassandraServer implements Cassandra.Iface } /* - * No-op since 2.2. + * No-op since 3.0. */ public void set_cql_version(String version) { @@ -2090,58 +2460,66 @@ public class CassandraServer implements Cassandra.Iface private static class ThriftCASRequest implements CASRequest { - private final ColumnFamily expected; - private final ColumnFamily updates; + private final CFMetaData metadata; + private final DecoratedKey key; - private ThriftCASRequest(ColumnFamily expected, ColumnFamily updates) + private final FilteredPartition expected; + private final PartitionUpdate updates; + + private ThriftCASRequest(FilteredPartition expected, PartitionUpdate updates) { + this.metadata = updates.metadata(); + this.key = updates.partitionKey(); this.expected = expected; this.updates = updates; } - public IDiskAtomFilter readFilter() + public SinglePartitionReadCommand readCommand(int nowInSec) { - return expected == null || expected.isEmpty() - ? new SliceQueryFilter(ColumnSlice.ALL_COLUMNS_ARRAY, false, 1) - : new NamesQueryFilter(ImmutableSortedSet.copyOf(expected.getComparator(), expected.getColumnNames())); + if (expected == null || expected.isEmpty()) + { + // We want to know if the partition exists, so just fetch a single cell. + ClusteringIndexSliceFilter filter = new ClusteringIndexSliceFilter(Slices.ALL, false); + DataLimits limits = DataLimits.thriftLimits(1, 1); + return new SinglePartitionSliceCommand(false, true, metadata, nowInSec, ColumnFilter.all(metadata), RowFilter.NONE, limits, key, filter); + } + + // Gather the clustering for the expected values and query those. + NavigableSet clusterings = new TreeSet<>(metadata.comparator); + for (Row row : expected) + clusterings.add(row.clustering().takeAlias()); + PartitionColumns columns = expected.staticRow().isEmpty() + ? metadata.partitionColumns().withoutStatics() + : metadata.partitionColumns(); + ClusteringIndexNamesFilter filter = new ClusteringIndexNamesFilter(clusterings, false); + return SinglePartitionReadCommand.create(true, metadata, nowInSec, ColumnFilter.selection(columns), RowFilter.NONE, DataLimits.NONE, key, filter); } - public boolean appliesTo(ColumnFamily current) + public boolean appliesTo(FilteredPartition current) { - long now = System.currentTimeMillis(); - - if (!hasLiveCells(expected, now)) - return !hasLiveCells(current, now); - else if (!hasLiveCells(current, now)) + if (expected == null || expected.isEmpty()) + return current.isEmpty(); + else if (current.isEmpty()) return false; - // current has been built from expected, so we know that it can't have columns - // that excepted don't have. So we just check that for each columns in expected: - // - if it is a tombstone, whether current has no column or a tombstone; - // - otherwise, that current has a live column with the same value. - for (Cell e : expected) + // Check that for everything we expected, the fetched values exists and correspond. + for (Row e : expected) { - Cell c = current.getColumn(e.name()); - if (e.isLive(now)) + Row c = current.getRow(e.clustering()); + if (c == null) + return false; + + for (Cell ce : e) { - if (c == null || !c.isLive(now) || !c.value().equals(e.value())) - return false; - } - else - { - if (c != null && c.isLive(now)) + Cell cc = c.getCell(ce.column()); + if (cc == null || !cc.value().equals(ce.value())) return false; } } return true; } - private static boolean hasLiveCells(ColumnFamily cf, long now) - { - return cf != null && !cf.hasOnlyTombstones(now); - } - - public ColumnFamily makeUpdates(ColumnFamily current) + public PartitionUpdate makeUpdates(FilteredPartition current) { return updates; } diff --git a/src/java/org/apache/cassandra/thrift/ThriftConversion.java b/src/java/org/apache/cassandra/thrift/ThriftConversion.java index adb925e2c8..148fd68b28 100644 --- a/src/java/org/apache/cassandra/thrift/ThriftConversion.java +++ b/src/java/org/apache/cassandra/thrift/ThriftConversion.java @@ -23,31 +23,35 @@ import java.util.*; import com.google.common.annotations.VisibleForTesting; import com.google.common.base.Strings; import com.google.common.collect.Maps; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; import org.apache.cassandra.cache.CachingOptions; import org.apache.cassandra.config.*; import org.apache.cassandra.cql3.ColumnIdentifier; import org.apache.cassandra.cql3.Operator; import org.apache.cassandra.cql3.UntypedResultSet; -import org.apache.cassandra.db.ColumnFamilyType; -import org.apache.cassandra.schema.LegacySchemaTables; -import org.apache.cassandra.db.WriteType; -import org.apache.cassandra.db.composites.CellNameType; -import org.apache.cassandra.db.composites.CellNames; +import org.apache.cassandra.db.*; +import org.apache.cassandra.db.filter.RowFilter; import org.apache.cassandra.db.marshal.*; import org.apache.cassandra.exceptions.*; import org.apache.cassandra.io.compress.CompressionParameters; import org.apache.cassandra.locator.AbstractReplicationStrategy; import org.apache.cassandra.locator.LocalStrategy; import org.apache.cassandra.serializers.MarshalException; +import org.apache.cassandra.schema.LegacySchemaTables; import org.apache.cassandra.utils.ByteBufferUtil; import org.apache.cassandra.utils.UUIDGen; /** * Static utility methods to convert internal structure to and from thrift ones. + * + * */ public class ThriftConversion { + private static final Logger logger = LoggerFactory.getLogger(ThriftConversion.class); + public static org.apache.cassandra.db.ConsistencyLevel fromThrift(ConsistencyLevel cl) { switch (cl) @@ -136,21 +140,14 @@ public class ThriftConversion return new TimedOutException(); } - public static List indexExpressionsFromThrift(List exprs) + public static RowFilter rowFilterFromThrift(CFMetaData metadata, List exprs) { - if (exprs == null) - return null; + if (exprs == null || exprs.isEmpty()) + return RowFilter.NONE; - if (exprs.isEmpty()) - return Collections.emptyList(); - - List converted = new ArrayList<>(exprs.size()); + RowFilter converted = RowFilter.forThrift(exprs.size()); for (IndexExpression expr : exprs) - { - converted.add(new org.apache.cassandra.db.IndexExpression(expr.column_name, - Operator.valueOf(expr.op.name()), - expr.value)); - } + converted.addThriftExpression(metadata, expr.column_name, Operator.valueOf(expr.op.name()), expr.value); return converted; } @@ -184,50 +181,75 @@ public class ThriftConversion public static CFMetaData fromThrift(CfDef cf_def) throws org.apache.cassandra.exceptions.InvalidRequestException, ConfigurationException { - return internalFromThrift(cf_def, Collections.emptyList()); + // This is a creation: the table is dense if it doesn't define any column_metadata + boolean isDense = cf_def.column_metadata == null || cf_def.column_metadata.isEmpty(); + return internalFromThrift(cf_def, true, Collections.emptyList(), isDense); } public static CFMetaData fromThriftForUpdate(CfDef cf_def, CFMetaData toUpdate) throws org.apache.cassandra.exceptions.InvalidRequestException, ConfigurationException { - return internalFromThrift(cf_def, toUpdate.allColumns()); + return internalFromThrift(cf_def, false, toUpdate.allColumns(), toUpdate.isDense()); } - // Convert a thrift CfDef, given a list of ColumnDefinitions to copy over to the created CFMetadata before the CQL metadata are rebuild - private static CFMetaData internalFromThrift(CfDef cf_def, Collection previousCQLMetadata) + private static boolean isSuper(String thriftColumnType) + throws org.apache.cassandra.exceptions.InvalidRequestException + { + switch (thriftColumnType.toLowerCase()) + { + case "standard": return false; + case "super": return true; + default: throw new org.apache.cassandra.exceptions.InvalidRequestException("Invalid column type " + thriftColumnType); + } + } + + /** + * Convert a thrift CfDef. + *

, + * This is used both for creation and update of CF. + * + * @param cf_def the thrift CfDef to convert. + * @param isCreation whether that is a new table creation or not. + * @param previousCQLMetadata if it is not a table creation, the previous + * definitions of the tables (which we use to preserve the CQL metadata). + * If it is a table creation, this will be empty. + * @param isDense whether the table is dense or not. + * + * @return the converted table definition. + */ + private static CFMetaData internalFromThrift(CfDef cf_def, + boolean isCreation, + Collection previousCQLMetadata, + boolean isDense) throws org.apache.cassandra.exceptions.InvalidRequestException, ConfigurationException { - ColumnFamilyType cfType = ColumnFamilyType.create(cf_def.column_type); - if (cfType == null) - throw new org.apache.cassandra.exceptions.InvalidRequestException("Invalid column type " + cf_def.column_type); - applyImplicitDefaults(cf_def); try { + boolean isSuper = isSuper(cf_def.column_type); AbstractType rawComparator = TypeParser.parse(cf_def.comparator_type); - AbstractType subComparator = cfType == ColumnFamilyType.Standard - ? null - : cf_def.subcomparator_type == null ? BytesType.instance : TypeParser.parse(cf_def.subcomparator_type); + AbstractType subComparator = isSuper + ? cf_def.subcomparator_type == null ? BytesType.instance : TypeParser.parse(cf_def.subcomparator_type) + : null; - AbstractType fullRawComparator = CFMetaData.makeRawAbstractType(rawComparator, subComparator); + AbstractType keyValidator = cf_def.isSetKey_validation_class() ? TypeParser.parse(cf_def.key_validation_class) : BytesType.instance; + AbstractType defaultValidator = TypeParser.parse(cf_def.default_validation_class); - AbstractType keyValidator = cf_def.isSetKey_validation_class() ? TypeParser.parse(cf_def.key_validation_class) : null; - - // Convert the REGULAR definitions from the input CfDef + // Convert the definitions from the input CfDef List defs = fromThrift(cf_def.keyspace, cf_def.name, rawComparator, subComparator, cf_def.column_metadata); - // Add the keyAlias if there is one, since that's on CQL metadata that thrift can actually change (for + // Add the keyAlias if there is one, since that's a CQL metadata that thrift can actually change (for // historical reasons) boolean hasKeyAlias = cf_def.isSetKey_alias() && keyValidator != null && !(keyValidator instanceof CompositeType); if (hasKeyAlias) - defs.add(ColumnDefinition.partitionKeyDef(cf_def.keyspace, cf_def.name, cf_def.key_alias, keyValidator, null)); + defs.add(ColumnDefinition.partitionKeyDef(cf_def.keyspace, cf_def.name, UTF8Type.instance.getString(cf_def.key_alias), keyValidator, null)); // Now add any CQL metadata that we want to copy, skipping the keyAlias if there was one for (ColumnDefinition def : previousCQLMetadata) { // isPartOfCellName basically means 'is not just a CQL metadata' - if (def.isPartOfCellName()) + if (def.isPartOfCellName(false, isSuper)) continue; if (def.kind == ColumnDefinition.Kind.PARTITION_KEY && hasKeyAlias) @@ -236,18 +258,25 @@ public class ThriftConversion defs.add(def); } - CellNameType comparator = CellNames.fromAbstractType(fullRawComparator, CFMetaData.calculateIsDense(fullRawComparator, defs)); - UUID cfId = Schema.instance.getId(cf_def.keyspace, cf_def.name); if (cfId == null) cfId = UUIDGen.getTimeUUID(); - CFMetaData newCFMD = new CFMetaData(cf_def.keyspace, cf_def.name, cfType, comparator, cfId); + boolean isCompound = isSuper ? false : (rawComparator instanceof CompositeType); + boolean isCounter = defaultValidator instanceof CounterColumnType; - newCFMD.addAllColumnDefinitions(defs); + // If it's a thrift table creation, adds the default CQL metadata for the new table + if (isCreation) + addDefaultCQLMetadata(defs, + cf_def.keyspace, + cf_def.name, + hasKeyAlias ? null : keyValidator, + rawComparator, + subComparator, + defaultValidator); + + CFMetaData newCFMD = CFMetaData.create(cf_def.keyspace, cf_def.name, cfId, isDense, isCompound, isSuper, isCounter, defs); - if (keyValidator != null) - newCFMD.keyValidator(keyValidator); if (cf_def.isSetGc_grace_seconds()) newCFMD.gcGraceSeconds(cf_def.gc_grace_seconds); if (cf_def.isSetMin_compaction_threshold()) @@ -280,9 +309,7 @@ public class ThriftConversion newCFMD.triggers(triggerDefinitionsFromThrift(cf_def.triggers)); return newCFMD.comment(cf_def.comment) - .defaultValidator(TypeParser.parse(cf_def.default_validation_class)) - .compressionParameters(CompressionParameters.create(cf_def.compression_options)) - .rebuild(); + .compressionParameters(CompressionParameters.create(cf_def.compression_options)); } catch (SyntaxException | MarshalException e) { @@ -290,6 +317,48 @@ public class ThriftConversion } } + private static void addDefaultCQLMetadata(Collection defs, + String ks, + String cf, + AbstractType keyValidator, + AbstractType comparator, + AbstractType subComparator, + AbstractType defaultValidator) + { + CompactTables.DefaultNames names = CompactTables.defaultNameGenerator(defs); + if (keyValidator != null) + { + if (keyValidator instanceof CompositeType) + { + List> subTypes = ((CompositeType)keyValidator).types; + for (int i = 0; i < subTypes.size(); i++) + defs.add(ColumnDefinition.partitionKeyDef(ks, cf, names.defaultPartitionKeyName(), subTypes.get(i), i)); + } + else + { + defs.add(ColumnDefinition.partitionKeyDef(ks, cf, names.defaultPartitionKeyName(), keyValidator, null)); + } + } + + if (subComparator != null) + { + // SuperColumn tables: we use a special map to hold dynamic values within a given super column + defs.add(ColumnDefinition.clusteringKeyDef(ks, cf, names.defaultClusteringName(), comparator, 0)); + defs.add(ColumnDefinition.regularDef(ks, cf, CompactTables.SUPER_COLUMN_MAP_COLUMN_STR, MapType.getInstance(subComparator, defaultValidator, true), null)); + } + else + { + List> subTypes = comparator instanceof CompositeType + ? ((CompositeType)comparator).types + : Collections.>singletonList(comparator); + + for (int i = 0; i < subTypes.size(); i++) + defs.add(ColumnDefinition.clusteringKeyDef(ks, cf, names.defaultClusteringName(), subTypes.get(i), i)); + + defs.add(ColumnDefinition.regularDef(ks, cf, names.defaultCompactValueName(), defaultValidator, null)); + } + } + /** applies implicit defaults to cf definition. useful in updates */ private static void applyImplicitDefaults(org.apache.cassandra.thrift.CfDef cf_def) { @@ -355,30 +424,30 @@ public class ThriftConversion public static CfDef toThrift(CFMetaData cfm) { CfDef def = new CfDef(cfm.ksName, cfm.cfName); - def.setColumn_type(cfm.cfType.name()); + def.setColumn_type(cfm.isSuper() ? "Super" : "Standard"); if (cfm.isSuper()) { def.setComparator_type(cfm.comparator.subtype(0).toString()); - def.setSubcomparator_type(cfm.comparator.subtype(1).toString()); + def.setSubcomparator_type(cfm.thriftColumnNameType().toString()); } else { - def.setComparator_type(cfm.comparator.toString()); + def.setComparator_type(LegacyLayout.makeLegacyComparator(cfm).toString()); } def.setComment(Strings.nullToEmpty(cfm.getComment())); def.setRead_repair_chance(cfm.getReadRepairChance()); def.setDclocal_read_repair_chance(cfm.getDcLocalReadRepairChance()); def.setGc_grace_seconds(cfm.getGcGraceSeconds()); - def.setDefault_validation_class(cfm.getDefaultValidator().toString()); + def.setDefault_validation_class(cfm.makeLegacyDefaultValidator().toString()); def.setKey_validation_class(cfm.getKeyValidator().toString()); def.setMin_compaction_threshold(cfm.getMinCompactionThreshold()); def.setMax_compaction_threshold(cfm.getMaxCompactionThreshold()); // We only return the alias if only one is set since thrift don't know about multiple key aliases if (cfm.partitionKeyColumns().size() == 1) def.setKey_alias(cfm.partitionKeyColumns().get(0).name.bytes); - def.setColumn_metadata(columnDefinitionsToThrift(cfm.allColumns())); + def.setColumn_metadata(columnDefinitionsToThrift(cfm, cfm.allColumns())); def.setCompaction_strategy(cfm.compactionStrategyClass.getName()); def.setCompaction_strategy_options(new HashMap<>(cfm.compactionStrategyOptions)); def.setCompression_options(cfm.compressionParameters.asThriftOptions()); @@ -402,8 +471,9 @@ public class ThriftConversion ColumnDef thriftColumnDef) throws SyntaxException, ConfigurationException { + boolean isSuper = thriftSubcomparator != null; // For super columns, the componentIndex is 1 because the ColumnDefinition applies to the column component. - Integer componentIndex = thriftSubcomparator != null ? 1 : null; + Integer componentIndex = isSuper ? 1 : null; AbstractType comparator = thriftSubcomparator == null ? thriftComparator : thriftSubcomparator; try { @@ -414,15 +484,18 @@ public class ThriftConversion throw new ConfigurationException(String.format("Column name %s is not valid for comparator %s", ByteBufferUtil.bytesToHex(thriftColumnDef.name), comparator)); } + // In our generic layout, we store thrift defined columns as static, but this doesn't work for super columns so we + // use a regular definition (and "dynamic" columns are handled in a map). + ColumnDefinition.Kind kind = isSuper ? ColumnDefinition.Kind.REGULAR : ColumnDefinition.Kind.STATIC; return new ColumnDefinition(ksName, cfName, - new ColumnIdentifier(ByteBufferUtil.clone(thriftColumnDef.name), comparator), + ColumnIdentifier.getInterned(ByteBufferUtil.clone(thriftColumnDef.name), comparator), TypeParser.parse(thriftColumnDef.validation_class), thriftColumnDef.index_type == null ? null : org.apache.cassandra.config.IndexType.valueOf(thriftColumnDef.index_type.name()), thriftColumnDef.index_options, thriftColumnDef.index_name, componentIndex, - ColumnDefinition.Kind.REGULAR); + kind); } private static List fromThrift(String ksName, @@ -456,11 +529,11 @@ public class ThriftConversion return cd; } - private static List columnDefinitionsToThrift(Collection columns) + private static List columnDefinitionsToThrift(CFMetaData metadata, Collection columns) { List thriftDefs = new ArrayList<>(columns.size()); for (ColumnDefinition def : columns) - if (def.kind == ColumnDefinition.Kind.REGULAR) + if (def.isPartOfCellName(metadata.isCQLTable(), metadata.isSuper())) thriftDefs.add(ThriftConversion.toThrift(def)); return thriftDefs; } diff --git a/src/java/org/apache/cassandra/thrift/ThriftResultsMerger.java b/src/java/org/apache/cassandra/thrift/ThriftResultsMerger.java new file mode 100644 index 0000000000..ccb6e74919 --- /dev/null +++ b/src/java/org/apache/cassandra/thrift/ThriftResultsMerger.java @@ -0,0 +1,317 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.cassandra.thrift; + +import java.util.Collections; +import java.util.Iterator; +import java.util.NoSuchElementException; + +import com.google.common.collect.AbstractIterator; +import com.google.common.collect.Iterators; +import com.google.common.collect.PeekingIterator; + +import org.apache.cassandra.config.CFMetaData; +import org.apache.cassandra.config.ColumnDefinition; +import org.apache.cassandra.db.*; +import org.apache.cassandra.db.rows.*; +import org.apache.cassandra.db.marshal.AbstractType; +import org.apache.cassandra.db.marshal.MapType; +import org.apache.cassandra.db.partitions.*; + +/** + * Given an iterator on a partition of a compact table, this return an iterator that merges the + * static row columns with the other results. + * + * Compact tables stores thrift column_metadata as static columns (see CompactTables for + * details). When reading for thrift however, we want to merge those static values with other + * results because: + * 1) on thrift, all "columns" are sorted together, whether or not they are declared + * column_metadata. + * 2) it's possible that a table add a value for a "dynamic" column, and later that column + * is statically defined. Merging "static" and "dynamic" columns make sure we don't miss + * a value prior to the column declaration. + * + * For example, if a thrift table declare 2 columns "c1" and "c5" and the results from a query + * is: + * Partition: static: { c1: 3, c5: 4 } + * "a" : { value : 2 } + * "c3": { value : 8 } + * "c7": { value : 1 } + * then this class transform it into: + * Partition: "a" : { value : 2 } + * "c1": { value : 3 } + * "c3": { value : 8 } + * "c5": { value : 4 } + * "c7": { value : 1 } + */ +public class ThriftResultsMerger extends WrappingUnfilteredPartitionIterator +{ + private final int nowInSec; + + private ThriftResultsMerger(UnfilteredPartitionIterator wrapped, int nowInSec) + { + super(wrapped); + this.nowInSec = nowInSec; + } + + public static UnfilteredPartitionIterator maybeWrap(UnfilteredPartitionIterator iterator, CFMetaData metadata, int nowInSec) + { + if (!metadata.isStaticCompactTable() && !metadata.isSuper()) + return iterator; + + return new ThriftResultsMerger(iterator, nowInSec); + } + + public static UnfilteredRowIterator maybeWrap(UnfilteredRowIterator iterator, int nowInSec) + { + if (!iterator.metadata().isStaticCompactTable() && !iterator.metadata().isSuper()) + return iterator; + + return iterator.metadata().isSuper() + ? new SuperColumnsPartitionMerger(iterator, nowInSec) + : new PartitionMerger(iterator, nowInSec); + } + + protected UnfilteredRowIterator computeNext(UnfilteredRowIterator iter) + { + return iter.metadata().isSuper() + ? new SuperColumnsPartitionMerger(iter, nowInSec) + : new PartitionMerger(iter, nowInSec); + } + + private static class PartitionMerger extends WrappingUnfilteredRowIterator + { + private final int nowInSec; + + // We initialize lazily to avoid having this iterator fetch the wrapped iterator before it's actually asked for it. + private boolean isInit; + + private Row staticRow; + private int i; // the index of the next column of static row to return + + private ReusableRow nextToMerge; + private Unfiltered nextFromWrapped; + + private PartitionMerger(UnfilteredRowIterator results, int nowInSec) + { + super(results); + assert results.metadata().isStaticCompactTable(); + this.nowInSec = nowInSec; + } + + private void init() + { + assert !isInit; + this.staticRow = super.staticRow(); + assert staticRow.columns().complexColumnCount() == 0; + + this.nextToMerge = createReusableRow(); + updateNextToMerge(); + isInit = true; + } + + @Override + public Row staticRow() + { + return Rows.EMPTY_STATIC_ROW; + } + + private ReusableRow createReusableRow() + { + return new ReusableRow(metadata().clusteringColumns().size(), metadata().partitionColumns().regulars, true, metadata().isCounter()); + } + + @Override + public boolean hasNext() + { + if (!isInit) + init(); + + return nextFromWrapped != null || nextToMerge != null || super.hasNext(); + } + + @Override + public Unfiltered next() + { + if (!isInit) + init(); + + if (nextFromWrapped == null && super.hasNext()) + nextFromWrapped = super.next(); + + if (nextFromWrapped == null) + { + if (nextToMerge == null) + throw new NoSuchElementException(); + + return consumeNextToMerge(); + } + + if (nextToMerge == null) + return consumeNextWrapped(); + + int cmp = metadata().comparator.compare(nextToMerge, nextFromWrapped); + if (cmp < 0) + return consumeNextToMerge(); + if (cmp > 0) + return consumeNextWrapped(); + + // Same row, but we know the row has only a single column so just pick the more recent + assert nextFromWrapped instanceof Row; + ReusableRow row = createReusableRow(); + Rows.merge((Row)consumeNextWrapped(), consumeNextToMerge(), columns().regulars, row.writer(), nowInSec); + return row; + } + + private Unfiltered consumeNextWrapped() + { + Unfiltered toReturn = nextFromWrapped; + nextFromWrapped = null; + return toReturn; + } + + private Row consumeNextToMerge() + { + Row toReturn = nextToMerge; + updateNextToMerge(); + return toReturn; + } + + private void updateNextToMerge() + { + while (i < staticRow.columns().simpleColumnCount()) + { + Cell cell = staticRow.getCell(staticRow.columns().getSimple(i++)); + if (cell != null) + { + // Given a static cell, the equivalent row uses the column name as clustering and the + // value as unique cell value. + Row.Writer writer = nextToMerge.writer(); + writer.writeClusteringValue(cell.column().name.bytes); + writer.writeCell(metadata().compactValueColumn(), cell.isCounterCell(), cell.value(), cell.livenessInfo(), cell.path()); + writer.endOfRow(); + return; + } + } + // Nothing more to merge. + nextToMerge = null; + } + } + + private static class SuperColumnsPartitionMerger extends WrappingUnfilteredRowIterator + { + private final int nowInSec; + private final ReusableRow reusableRow; + private final ColumnDefinition superColumnMapColumn; + private final AbstractType columnComparator; + + private SuperColumnsPartitionMerger(UnfilteredRowIterator results, int nowInSec) + { + super(results); + assert results.metadata().isSuper(); + this.nowInSec = nowInSec; + + this.superColumnMapColumn = results.metadata().compactValueColumn(); + assert superColumnMapColumn != null && superColumnMapColumn.type instanceof MapType; + + this.reusableRow = new ReusableRow(results.metadata().clusteringColumns().size(), + Columns.of(superColumnMapColumn), + true, + results.metadata().isCounter()); + this.columnComparator = ((MapType)superColumnMapColumn.type).nameComparator(); + } + + @Override + public Unfiltered next() + { + Unfiltered next = super.next(); + if (next.kind() != Unfiltered.Kind.ROW) + return next; + + Row row = (Row)next; + Row.Writer writer = reusableRow.writer(); + row.clustering().writeTo(writer); + + PeekingIterator staticCells = Iterators.peekingIterator(makeStaticCellIterator(row)); + if (!staticCells.hasNext()) + return row; + + Iterator cells = row.getCells(superColumnMapColumn); + PeekingIterator dynamicCells = Iterators.peekingIterator(cells.hasNext() ? cells : Collections.emptyIterator()); + + while (staticCells.hasNext() && dynamicCells.hasNext()) + { + Cell staticCell = staticCells.peek(); + Cell dynamicCell = dynamicCells.peek(); + int cmp = columnComparator.compare(staticCell.column().name.bytes, dynamicCell.path().get(0)); + if (cmp < 0) + { + staticCell = staticCells.next(); + writer.writeCell(superColumnMapColumn, staticCell.isCounterCell(), staticCell.value(), staticCell.livenessInfo(), CellPath.create(staticCell.column().name.bytes)); + } + else if (cmp > 0) + { + dynamicCells.next().writeTo(writer); + } + else + { + staticCell = staticCells.next(); + Cell toMerge = Cells.create(superColumnMapColumn, + staticCell.isCounterCell(), + staticCell.value(), + staticCell.livenessInfo(), + CellPath.create(staticCell.column().name.bytes)); + Cells.reconcile(toMerge, dynamicCells.next(), nowInSec).writeTo(writer); + } + } + + while (staticCells.hasNext()) + { + Cell staticCell = staticCells.next(); + writer.writeCell(superColumnMapColumn, staticCell.isCounterCell(), staticCell.value(), staticCell.livenessInfo(), CellPath.create(staticCell.column().name.bytes)); + } + while (dynamicCells.hasNext()) + { + dynamicCells.next().writeTo(writer); + } + + writer.endOfRow(); + return reusableRow; + } + + private static Iterator makeStaticCellIterator(final Row row) + { + return new AbstractIterator() + { + private int i; + + protected Cell computeNext() + { + while (i < row.columns().simpleColumnCount()) + { + Cell cell = row.getCell(row.columns().getSimple(i++)); + if (cell != null) + return cell; + } + return endOfData(); + } + }; + } + } +} + diff --git a/src/java/org/apache/cassandra/thrift/ThriftValidation.java b/src/java/org/apache/cassandra/thrift/ThriftValidation.java index 376895229e..dd5bf9885f 100644 --- a/src/java/org/apache/cassandra/thrift/ThriftValidation.java +++ b/src/java/org/apache/cassandra/thrift/ThriftValidation.java @@ -24,17 +24,10 @@ import org.slf4j.Logger; import org.slf4j.LoggerFactory; import org.apache.cassandra.config.*; -import org.apache.cassandra.cql3.ColumnIdentifier; +import org.apache.cassandra.cql3.Attributes; import org.apache.cassandra.db.*; -import org.apache.cassandra.db.composites.*; -import org.apache.cassandra.db.filter.IDiskAtomFilter; -import org.apache.cassandra.db.filter.NamesQueryFilter; -import org.apache.cassandra.db.filter.SliceQueryFilter; -import org.apache.cassandra.db.index.SecondaryIndex; import org.apache.cassandra.db.index.SecondaryIndexManager; import org.apache.cassandra.db.marshal.AbstractType; -import org.apache.cassandra.db.marshal.ColumnToCollectionType; -import org.apache.cassandra.db.marshal.UTF8Type; import org.apache.cassandra.dht.IPartitioner; import org.apache.cassandra.dht.Token; import org.apache.cassandra.serializers.MarshalException; @@ -123,7 +116,7 @@ public class ThriftValidation */ public static void validateColumnPath(CFMetaData metadata, ColumnPath column_path) throws org.apache.cassandra.exceptions.InvalidRequestException { - if (metadata.cfType == ColumnFamilyType.Standard) + if (!metadata.isSuper()) { if (column_path.super_column != null) { @@ -151,7 +144,7 @@ public class ThriftValidation public static void validateColumnParent(CFMetaData metadata, ColumnParent column_parent) throws org.apache.cassandra.exceptions.InvalidRequestException { - if (metadata.cfType == ColumnFamilyType.Standard) + if (!metadata.isSuper()) { if (column_parent.super_column != null) { @@ -168,14 +161,7 @@ public class ThriftValidation // column_path_or_parent is a ColumnPath for remove, where the "column" is optional even for a standard CF static void validateColumnPathOrParent(CFMetaData metadata, ColumnPath column_path_or_parent) throws org.apache.cassandra.exceptions.InvalidRequestException { - if (metadata.cfType == ColumnFamilyType.Standard) - { - if (column_path_or_parent.super_column != null) - { - throw new org.apache.cassandra.exceptions.InvalidRequestException("supercolumn may not be specified for standard CF " + metadata.cfName); - } - } - if (metadata.cfType == ColumnFamilyType.Super) + if (metadata.isSuper()) { if (column_path_or_parent.super_column == null && column_path_or_parent.column != null) { @@ -183,6 +169,13 @@ public class ThriftValidation + metadata.cfName); } } + else + { + if (column_path_or_parent.super_column != null) + { + throw new org.apache.cassandra.exceptions.InvalidRequestException("supercolumn may not be specified for standard CF " + metadata.cfName); + } + } if (column_path_or_parent.column != null) { validateColumnNames(metadata, column_path_or_parent.super_column, Arrays.asList(column_path_or_parent.column)); @@ -193,13 +186,30 @@ public class ThriftValidation } } + private static AbstractType getThriftColumnNameComparator(CFMetaData metadata, ByteBuffer superColumnName) + { + if (!metadata.isSuper()) + return LegacyLayout.makeLegacyComparator(metadata); + + if (superColumnName == null) + { + // comparator for super column name + return metadata.comparator.subtype(0); + } + else + { + // comparator for sub columns + return metadata.thriftColumnNameType(); + } + } + /** * Validates the column names but not the parent path or data */ private static void validateColumnNames(CFMetaData metadata, ByteBuffer superColumnName, Iterable column_names) throws org.apache.cassandra.exceptions.InvalidRequestException { - int maxNameLength = Cell.MAX_NAME_LENGTH; + int maxNameLength = LegacyLayout.MAX_CELL_NAME_LENGTH; if (superColumnName != null) { @@ -207,10 +217,10 @@ public class ThriftValidation throw new org.apache.cassandra.exceptions.InvalidRequestException("supercolumn name length must not be greater than " + maxNameLength); if (superColumnName.remaining() == 0) throw new org.apache.cassandra.exceptions.InvalidRequestException("supercolumn name must not be empty"); - if (metadata.cfType == ColumnFamilyType.Standard) + if (!metadata.isSuper()) throw new org.apache.cassandra.exceptions.InvalidRequestException("supercolumn specified to table " + metadata.cfName + " containing normal columns"); } - AbstractType comparator = SuperColumns.getComparatorFor(metadata, superColumnName); + AbstractType comparator = getThriftColumnNameComparator(metadata, superColumnName); boolean isCQL3Table = !metadata.isThriftCompatible(); for (ByteBuffer name : column_names) { @@ -229,31 +239,26 @@ public class ThriftValidation if (isCQL3Table) { - // CQL3 table don't support having only part of their composite column names set - Composite composite = metadata.comparator.fromByteBuffer(name); - - int minComponents = metadata.comparator.clusteringPrefixSize() + 1; - if (composite.size() < minComponents) - throw new org.apache.cassandra.exceptions.InvalidRequestException(String.format("Not enough components (found %d but %d expected) for column name since %s is a CQL3 table", - composite.size(), minComponents, metadata.cfName)); - - // Furthermore, the column name must be a declared one. - int columnIndex = metadata.comparator.clusteringPrefixSize(); - ByteBuffer CQL3ColumnName = composite.get(columnIndex); - if (!CQL3ColumnName.hasRemaining()) - continue; // Row marker, ok - - ColumnIdentifier columnId = new ColumnIdentifier(CQL3ColumnName, metadata.comparator.subtype(columnIndex)); - if (metadata.getColumnDefinition(columnId) == null) - throw new org.apache.cassandra.exceptions.InvalidRequestException(String.format("Invalid cell for CQL3 table %s. The CQL3 column component (%s) does not correspond to a defined CQL3 column", - metadata.cfName, columnId)); - - // On top of that, if we have a collection component, he (CQL3) column must be a collection - if (metadata.comparator.hasCollections() && composite.size() == metadata.comparator.size()) + try { - ColumnToCollectionType collectionType = metadata.comparator.collectionType(); - if (!collectionType.defined.containsKey(CQL3ColumnName)) - throw new org.apache.cassandra.exceptions.InvalidRequestException(String.format("Invalid collection component, %s is not a collection", UTF8Type.instance.getString(CQL3ColumnName))); + LegacyLayout.LegacyCellName cname = LegacyLayout.decodeCellName(metadata, name); + assert cname.clustering.size() == metadata.comparator.size(); + + // CQL3 table don't support having only part of their composite column names set + for (int i = 0; i < cname.clustering.size(); i++) + { + if (cname.clustering.get(i) == null) + throw new org.apache.cassandra.exceptions.InvalidRequestException(String.format("Not enough components (found %d but %d expected) for column name since %s is a CQL3 table", + i, metadata.comparator.size() + 1, metadata.cfName)); + } + + // On top of that, if we have a collection component, the (CQL3) column must be a collection + if (cname.column != null && cname.collectionElement != null && !cname.column.type.isCollection()) + throw new org.apache.cassandra.exceptions.InvalidRequestException(String.format("Invalid collection component, %s is not a collection", cname.column.name)); + } + catch (IllegalArgumentException | UnknownColumnException e) + { + throw new org.apache.cassandra.exceptions.InvalidRequestException(String.format("Error validating cell name for CQL3 table %s: %s", metadata.cfName, e.getMessage())); } } } @@ -269,13 +274,13 @@ public class ThriftValidation if (range.count < 0) throw new org.apache.cassandra.exceptions.InvalidRequestException("get_slice requires non-negative count"); - int maxNameLength = Cell.MAX_NAME_LENGTH; + int maxNameLength = LegacyLayout.MAX_CELL_NAME_LENGTH; if (range.start.remaining() > maxNameLength) throw new org.apache.cassandra.exceptions.InvalidRequestException("range start length cannot be larger than " + maxNameLength); if (range.finish.remaining() > maxNameLength) throw new org.apache.cassandra.exceptions.InvalidRequestException("range finish length cannot be larger than " + maxNameLength); - AbstractType comparator = SuperColumns.getComparatorFor(metadata, column_parent.super_column); + AbstractType comparator = getThriftColumnNameComparator(metadata, column_parent.super_column); try { comparator.validate(range.start); @@ -295,7 +300,7 @@ public class ThriftValidation } } - public static void validateColumnOrSuperColumn(CFMetaData metadata, ByteBuffer key, ColumnOrSuperColumn cosc) + public static void validateColumnOrSuperColumn(CFMetaData metadata, ColumnOrSuperColumn cosc) throws org.apache.cassandra.exceptions.InvalidRequestException { boolean isCommutative = metadata.isCounter(); @@ -316,7 +321,7 @@ public class ThriftValidation validateTtl(cosc.column); validateColumnPath(metadata, new ColumnPath(metadata.cfName).setSuper_column((ByteBuffer)null).setColumn(cosc.column.name)); - validateColumnData(metadata, key, null, cosc.column); + validateColumnData(metadata, null, cosc.column); } if (cosc.super_column != null) @@ -327,7 +332,7 @@ public class ThriftValidation for (Column c : cosc.super_column.columns) { validateColumnPath(metadata, new ColumnPath(metadata.cfName).setSuper_column(cosc.super_column.name).setColumn(c.name)); - validateColumnData(metadata, key, cosc.super_column.name, c); + validateColumnData(metadata, cosc.super_column.name, c); } } @@ -356,8 +361,8 @@ public class ThriftValidation if (column.ttl <= 0) throw new org.apache.cassandra.exceptions.InvalidRequestException("ttl must be positive"); - if (column.ttl > ExpiringCell.MAX_TTL) - throw new org.apache.cassandra.exceptions.InvalidRequestException(String.format("ttl is too large. requested (%d) maximum (%d)", column.ttl, ExpiringCell.MAX_TTL)); + if (column.ttl > Attributes.MAX_TTL) + throw new org.apache.cassandra.exceptions.InvalidRequestException(String.format("ttl is too large. requested (%d) maximum (%d)", column.ttl, Attributes.MAX_TTL)); } else { @@ -366,7 +371,7 @@ public class ThriftValidation } } - public static void validateMutation(CFMetaData metadata, ByteBuffer key, Mutation mut) + public static void validateMutation(CFMetaData metadata, Mutation mut) throws org.apache.cassandra.exceptions.InvalidRequestException { ColumnOrSuperColumn cosc = mut.column_or_supercolumn; @@ -383,7 +388,7 @@ public class ThriftValidation if (cosc != null) { - validateColumnOrSuperColumn(metadata, key, cosc); + validateColumnOrSuperColumn(metadata, cosc); } else { @@ -400,7 +405,7 @@ public class ThriftValidation if (del.predicate != null) validateSlicePredicate(metadata, del.super_column, del.predicate); - if (metadata.cfType == ColumnFamilyType.Standard && del.super_column != null) + if (!metadata.isSuper() && del.super_column != null) { String msg = String.format("Deletion of super columns is not possible on a standard table (KeySpace=%s Table=%s Deletion=%s)", metadata.ksName, metadata.cfName, del); throw new org.apache.cassandra.exceptions.InvalidRequestException(msg); @@ -409,7 +414,7 @@ public class ThriftValidation if (metadata.isCounter()) { // forcing server timestamp even if a timestamp was set for coherence with other counter operation - del.timestamp = System.currentTimeMillis(); + del.timestamp = FBUtilities.timestampMicros(); } else if (!del.isSetTimestamp()) { @@ -432,7 +437,7 @@ public class ThriftValidation /** * Validates the data part of the column (everything in the column object but the name, which is assumed to be valid) */ - public static void validateColumnData(CFMetaData metadata, ByteBuffer key, ByteBuffer scName, Column column) throws org.apache.cassandra.exceptions.InvalidRequestException + public static void validateColumnData(CFMetaData metadata, ByteBuffer scName, Column column) throws org.apache.cassandra.exceptions.InvalidRequestException { validateTtl(column); if (!column.isSetValue()) @@ -440,14 +445,17 @@ public class ThriftValidation if (!column.isSetTimestamp()) throw new org.apache.cassandra.exceptions.InvalidRequestException("Column timestamp is required"); - CellName cn = scName == null - ? metadata.comparator.cellFromByteBuffer(column.name) - : metadata.comparator.makeCellName(scName, column.name); try { - AbstractType validator = metadata.getValueValidator(cn); - if (validator != null) - validator.validate(column.value); + LegacyLayout.LegacyCellName cn = LegacyLayout.decodeCellName(metadata, scName, column.name); + cn.column.validateCellValue(column.value); + + // Indexed column values cannot be larger than 64K. See CASSANDRA-3057/4240 for more details + Keyspace.open(metadata.ksName).getColumnFamilyStore(metadata.cfName).indexManager.validate(cn.column, column.value, null); + } + catch (UnknownColumnException e) + { + throw new org.apache.cassandra.exceptions.InvalidRequestException(e.getMessage()); } catch (MarshalException me) { @@ -458,25 +466,9 @@ public class ThriftValidation me.getMessage(), metadata.ksName, metadata.cfName, - (SuperColumns.getComparatorFor(metadata, scName != null)).getString(column.name))); + (getThriftColumnNameComparator(metadata, scName)).getString(column.name))); } - // Indexed column values cannot be larger than 64K. See CASSANDRA-3057/4240 for more details - SecondaryIndex failedIndex = Keyspace.open(metadata.ksName).getColumnFamilyStore(metadata.cfName).indexManager.validate(key, asDBColumn(cn, column)); - if (failedIndex != null) - throw new org.apache.cassandra.exceptions.InvalidRequestException(String.format("Can't index column value of size %d for index %s in CF %s of KS %s", - column.value.remaining(), - failedIndex.getIndexName(), - metadata.cfName, - metadata.ksName)); - } - - private static Cell asDBColumn(CellName name, Column column) - { - if (column.ttl <= 0) - return new BufferCell(name, column.value, column.timestamp); - else - return new BufferExpiringCell(name, column.value, column.timestamp, column.ttl); } /** @@ -535,8 +527,8 @@ public class ThriftValidation else if (range.start_key != null && range.end_token != null) { // start_token/end_token can wrap, but key/token should not - RowPosition stop = p.getTokenFactory().fromString(range.end_token).maxKeyBound(); - if (RowPosition.ForKey.get(range.start_key, p).compareTo(stop) > 0 && !stop.isMinimum()) + PartitionPosition stop = p.getTokenFactory().fromString(range.end_token).maxKeyBound(); + if (PartitionPosition.ForKey.get(range.start_key, p).compareTo(stop) > 0 && !stop.isMinimum()) throw new org.apache.cassandra.exceptions.InvalidRequestException("Start key's token sorts after end token"); } @@ -577,7 +569,7 @@ public class ThriftValidation return false; SecondaryIndexManager idxManager = Keyspace.open(metadata.ksName).getColumnFamilyStore(metadata.cfName).indexManager; - AbstractType nameValidator = SuperColumns.getComparatorFor(metadata, null); + AbstractType nameValidator = getThriftColumnNameComparator(metadata, null); boolean isIndexed = false; for (IndexExpression expression : index_clause) @@ -597,11 +589,18 @@ public class ThriftValidation if (expression.value.remaining() > 0xFFFF) throw new org.apache.cassandra.exceptions.InvalidRequestException("Index expression values may not be larger than 64K"); - CellName name = metadata.comparator.cellFromByteBuffer(expression.column_name); - AbstractType valueValidator = metadata.getValueValidator(name); + ColumnDefinition def = metadata.getColumnDefinition(expression.column_name); + if (def == null) + { + if (!metadata.isCompactTable()) + throw new org.apache.cassandra.exceptions.InvalidRequestException(String.format("Unknown column %s", nameValidator.getString(expression.column_name))); + + def = metadata.compactValueColumn(); + } + try { - valueValidator.validate(expression.value); + def.type.validate(expression.value); } catch (MarshalException me) { @@ -611,7 +610,7 @@ public class ThriftValidation me.getMessage())); } - isIndexed |= (expression.op == IndexOperator.EQ) && idxManager.indexes(name); + isIndexed |= (expression.op == IndexOperator.EQ) && idxManager.indexes(def); } return isIndexed; @@ -637,32 +636,32 @@ public class ThriftValidation throw new org.apache.cassandra.exceptions.InvalidRequestException("system keyspace is not user-modifiable"); } - public static IDiskAtomFilter asIFilter(SlicePredicate sp, CFMetaData metadata, ByteBuffer superColumn) - { - SliceRange sr = sp.slice_range; - IDiskAtomFilter filter; + //public static IDiskAtomFilter asIFilter(SlicePredicate sp, CFMetaData metadata, ByteBuffer superColumn) + //{ + // SliceRange sr = sp.slice_range; + // IDiskAtomFilter filter; - CellNameType comparator = metadata.isSuper() - ? new SimpleDenseCellNameType(metadata.comparator.subtype(superColumn == null ? 0 : 1)) - : metadata.comparator; - if (sr == null) - { + // CellNameType comparator = metadata.isSuper() + // ? new SimpleDenseCellNameType(metadata.comparator.subtype(superColumn == null ? 0 : 1)) + // : metadata.comparator; + // if (sr == null) + // { - SortedSet ss = new TreeSet(comparator); - for (ByteBuffer bb : sp.column_names) - ss.add(comparator.cellFromByteBuffer(bb)); - filter = new NamesQueryFilter(ss); - } - else - { - filter = new SliceQueryFilter(comparator.fromByteBuffer(sr.start), - comparator.fromByteBuffer(sr.finish), - sr.reversed, - sr.count); - } + // SortedSet ss = new TreeSet(comparator); + // for (ByteBuffer bb : sp.column_names) + // ss.add(comparator.cellFromByteBuffer(bb)); + // filter = new NamesQueryFilter(ss); + // } + // else + // { + // filter = new SliceQueryFilter(comparator.fromByteBuffer(sr.start), + // comparator.fromByteBuffer(sr.finish), + // sr.reversed, + // sr.count); + // } - if (metadata.isSuper()) - filter = SuperColumns.fromSCFilter(metadata.comparator, superColumn, filter); - return filter; - } + // if (metadata.isSuper()) + // filter = SuperColumns.fromSCFilter(metadata.comparator, superColumn, filter); + // return filter; + //} } diff --git a/src/java/org/apache/cassandra/tools/SSTableExport.java b/src/java/org/apache/cassandra/tools/SSTableExport.java deleted file mode 100644 index 9f833e7f6e..0000000000 --- a/src/java/org/apache/cassandra/tools/SSTableExport.java +++ /dev/null @@ -1,480 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.apache.cassandra.tools; - -import java.io.File; -import java.io.IOException; -import java.io.PrintStream; -import java.util.*; - -import org.apache.cassandra.io.sstable.format.SSTableReader; -import org.apache.commons.cli.*; - -import org.apache.cassandra.config.CFMetaData; -import org.apache.cassandra.config.Schema; -import org.apache.cassandra.db.*; -import org.apache.cassandra.db.composites.CellNameType; -import org.apache.cassandra.db.marshal.AbstractType; -import org.apache.cassandra.dht.IPartitioner; -import org.apache.cassandra.exceptions.ConfigurationException; -import org.apache.cassandra.io.sstable.*; -import org.apache.cassandra.io.util.RandomAccessReader; -import org.apache.cassandra.utils.ByteBufferUtil; -import org.codehaus.jackson.JsonGenerator; -import org.codehaus.jackson.map.ObjectMapper; - -/** - * Export SSTables to JSON format. - */ -public class SSTableExport -{ - private static final ObjectMapper jsonMapper = new ObjectMapper(); - - private static final String KEY_OPTION = "k"; - private static final String EXCLUDEKEY_OPTION = "x"; - private static final String ENUMERATEKEYS_OPTION = "e"; - - private static final Options options = new Options(); - private static CommandLine cmd; - - static - { - Option optKey = new Option(KEY_OPTION, true, "Row key"); - // Number of times -k can be passed on the command line. - optKey.setArgs(500); - options.addOption(optKey); - - Option excludeKey = new Option(EXCLUDEKEY_OPTION, true, "Excluded row key"); - // Number of times -x can be passed on the command line. - excludeKey.setArgs(500); - options.addOption(excludeKey); - - Option optEnumerate = new Option(ENUMERATEKEYS_OPTION, false, "enumerate keys only"); - options.addOption(optEnumerate); - - // disabling auto close of the stream - jsonMapper.configure(JsonGenerator.Feature.AUTO_CLOSE_TARGET, false); - } - - /** - * Checks if PrintStream error and throw exception - * - * @param out The PrintStream to be check - */ - private static void checkStream(PrintStream out) throws IOException - { - if (out.checkError()) - throw new IOException("Error writing output stream"); - } - - /** - * JSON Hash Key serializer - * - * @param out The output steam to write data - * @param value value to set as a key - */ - private static void writeKey(PrintStream out, String value) - { - writeJSON(out, value); - out.print(": "); - } - - private static List serializeAtom(OnDiskAtom atom, CFMetaData cfMetaData) - { - if (atom instanceof Cell) - { - return serializeColumn((Cell) atom, cfMetaData); - } - else - { - assert atom instanceof RangeTombstone; - RangeTombstone rt = (RangeTombstone) atom; - ArrayList serializedColumn = new ArrayList(); - serializedColumn.add(cfMetaData.comparator.getString(rt.min)); - serializedColumn.add(cfMetaData.comparator.getString(rt.max)); - serializedColumn.add(rt.data.markedForDeleteAt); - serializedColumn.add("t"); - serializedColumn.add(rt.data.localDeletionTime); - return serializedColumn; - } - } - - /** - * Serialize a given cell to a List of Objects that jsonMapper knows how to turn into strings. Type is - * - * human_readable_name, value, timestamp, [flag, [options]] - * - * Value is normally the human readable value as rendered by the validator, but for deleted cells we - * give the local deletion time instead. - * - * Flag may be exactly one of {d,e,c} for deleted, expiring, or counter: - * - No options for deleted cells - * - If expiring, options will include the TTL and local deletion time. - * - If counter, options will include timestamp of last delete - * - * @param cell cell presentation - * @param cfMetaData Column Family metadata (to get validator) - * @return cell as serialized list - */ - private static List serializeColumn(Cell cell, CFMetaData cfMetaData) - { - CellNameType comparator = cfMetaData.comparator; - ArrayList serializedColumn = new ArrayList(); - - serializedColumn.add(comparator.getString(cell.name())); - - if (cell instanceof DeletedCell) - { - serializedColumn.add(cell.getLocalDeletionTime()); - } - else - { - AbstractType validator = cfMetaData.getValueValidator(cell.name()); - serializedColumn.add(validator.getString(cell.value())); - } - - serializedColumn.add(cell.timestamp()); - - if (cell instanceof DeletedCell) - { - serializedColumn.add("d"); - } - else if (cell instanceof ExpiringCell) - { - serializedColumn.add("e"); - serializedColumn.add(((ExpiringCell) cell).getTimeToLive()); - serializedColumn.add(cell.getLocalDeletionTime()); - } - else if (cell instanceof CounterCell) - { - serializedColumn.add("c"); - serializedColumn.add(((CounterCell) cell).timestampOfLastDelete()); - } - - return serializedColumn; - } - - /** - * Get portion of the columns and serialize in loop while not more columns left in the row - * - * @param row SSTableIdentityIterator row representation with Column Family - * @param key Decorated Key for the required row - * @param out output stream - */ - private static void serializeRow(SSTableIdentityIterator row, DecoratedKey key, PrintStream out) - { - serializeRow(row.getColumnFamily().deletionInfo(), row, row.getColumnFamily().metadata(), key, out); - } - - private static void serializeRow(DeletionInfo deletionInfo, Iterator atoms, CFMetaData metadata, DecoratedKey key, PrintStream out) - { - out.print("{"); - writeKey(out, "key"); - writeJSON(out, metadata.getKeyValidator().getString(key.getKey())); - out.print(",\n"); - - if (!deletionInfo.isLive()) - { - out.print(" "); - writeKey(out, "metadata"); - out.print("{"); - writeKey(out, "deletionInfo"); - writeJSON(out, deletionInfo.getTopLevelDeletion()); - out.print("}"); - out.print(",\n"); - } - - out.print(" "); - writeKey(out, "cells"); - out.print("["); - while (atoms.hasNext()) - { - writeJSON(out, serializeAtom(atoms.next(), metadata)); - - if (atoms.hasNext()) - out.print(",\n "); - } - out.print("]"); - - out.print("}"); - } - - /** - * Enumerate row keys from an SSTableReader and write the result to a PrintStream. - * - * @param desc the descriptor of the file to export the rows from - * @param outs PrintStream to write the output to - * @param metadata Metadata to print keys in a proper format - * @throws IOException on failure to read/write input/output - */ - public static void enumeratekeys(Descriptor desc, PrintStream outs, CFMetaData metadata) - throws IOException - { - try (KeyIterator iter = new KeyIterator(desc)) - { - DecoratedKey lastKey = null; - while (iter.hasNext()) - { - DecoratedKey key = iter.next(); - - // validate order of the keys in the sstable - if (lastKey != null && lastKey.compareTo(key) > 0) - throw new IOException("Key out of order! " + lastKey + " > " + key); - lastKey = key; - - outs.println(metadata.getKeyValidator().getString(key.getKey())); - checkStream(outs); // flushes - } - } - } - - /** - * Export specific rows from an SSTable and write the resulting JSON to a PrintStream. - * - * @param desc the descriptor of the sstable to read from - * @param outs PrintStream to write the output to - * @param toExport the keys corresponding to the rows to export - * @param excludes keys to exclude from export - * @param metadata Metadata to print keys in a proper format - * @throws IOException on failure to read/write input/output - */ - public static void export(Descriptor desc, PrintStream outs, Collection toExport, String[] excludes, CFMetaData metadata) throws IOException - { - SSTableReader sstable = SSTableReader.open(desc); - - try (RandomAccessReader dfile = sstable.openDataReader()) - { - IPartitioner partitioner = sstable.partitioner; - - if (excludes != null) - toExport.removeAll(Arrays.asList(excludes)); - - outs.println("["); - - int i = 0; - - // last key to compare order - DecoratedKey lastKey = null; - - for (String key : toExport) - { - DecoratedKey decoratedKey = partitioner.decorateKey(metadata.getKeyValidator().fromString(key)); - - if (lastKey != null && lastKey.compareTo(decoratedKey) > 0) - throw new IOException("Key out of order! " + lastKey + " > " + decoratedKey); - - lastKey = decoratedKey; - - RowIndexEntry entry = sstable.getPosition(decoratedKey, SSTableReader.Operator.EQ); - if (entry == null) - continue; - - dfile.seek(entry.position); - ByteBufferUtil.readWithShortLength(dfile); // row key - DeletionInfo deletionInfo = new DeletionInfo(DeletionTime.serializer.deserialize(dfile)); - - Iterator atomIterator = sstable.metadata.getOnDiskIterator(dfile, sstable.descriptor.version); - checkStream(outs); - - if (i != 0) - outs.println(","); - i++; - serializeRow(deletionInfo, atomIterator, sstable.metadata, decoratedKey, outs); - } - - outs.println("\n]"); - outs.flush(); - } - } - - // This is necessary to accommodate the test suite since you cannot open a Reader more - // than once from within the same process. - static void export(SSTableReader reader, PrintStream outs, String[] excludes) throws IOException - { - Set excludeSet = new HashSet(); - - if (excludes != null) - excludeSet = new HashSet<>(Arrays.asList(excludes)); - - SSTableIdentityIterator row; - ISSTableScanner scanner = reader.getScanner(); - try - { - outs.println("["); - - int i = 0; - - // collecting keys to export - while (scanner.hasNext()) - { - row = (SSTableIdentityIterator) scanner.next(); - - String currentKey = row.getColumnFamily().metadata().getKeyValidator().getString(row.getKey().getKey()); - - if (excludeSet.contains(currentKey)) - continue; - else if (i != 0) - outs.println(","); - - serializeRow(row, row.getKey(), outs); - checkStream(outs); - - i++; - } - - outs.println("\n]"); - outs.flush(); - } - finally - { - scanner.close(); - } - } - - /** - * Export an SSTable and write the resulting JSON to a PrintStream. - * - * @param desc the descriptor of the sstable to read from - * @param outs PrintStream to write the output to - * @param excludes keys to exclude from export - * @throws IOException on failure to read/write input/output - */ - public static void export(Descriptor desc, PrintStream outs, String[] excludes) throws IOException - { - export(SSTableReader.open(desc), outs, excludes); - } - - /** - * Export an SSTable and write the resulting JSON to standard out. - * - * @param desc the descriptor of the sstable to read from - * @param excludes keys to exclude from export - * @throws IOException on failure to read/write SSTable/standard out - */ - public static void export(Descriptor desc, String[] excludes) throws IOException - { - export(desc, System.out, excludes); - } - - /** - * Given arguments specifying an SSTable, and optionally an output file, - * export the contents of the SSTable to JSON. - * - * @param args command lines arguments - * @throws ConfigurationException on configuration failure (wrong params given) - */ - public static void main(String[] args) throws ConfigurationException - { - System.err.println("WARNING: please note that sstable2json is now deprecated and will be removed in Cassandra 3.0. " - + "Please see https://issues.apache.org/jira/browse/CASSANDRA-9618 for details."); - - String usage = String.format("Usage: %s [-k key [-k key [...]] -x key [-x key [...]]]%n", SSTableExport.class.getName()); - - CommandLineParser parser = new PosixParser(); - try - { - cmd = parser.parse(options, args); - } - catch (ParseException e1) - { - System.err.println(e1.getMessage()); - System.err.println(usage); - System.exit(1); - } - - - if (cmd.getArgs().length != 1) - { - System.err.println("You must supply exactly one sstable"); - System.err.println(usage); - System.exit(1); - } - - - String[] keys = cmd.getOptionValues(KEY_OPTION); - String[] excludes = cmd.getOptionValues(EXCLUDEKEY_OPTION); - String ssTableFileName = new File(cmd.getArgs()[0]).getAbsolutePath(); - - Schema.instance.loadFromDisk(false); - Descriptor descriptor = Descriptor.fromFilename(ssTableFileName); - - // Start by validating keyspace name - if (Schema.instance.getKSMetaData(descriptor.ksname) == null) - { - System.err.println(String.format("Filename %s references to nonexistent keyspace: %s!", - ssTableFileName, descriptor.ksname)); - System.exit(1); - } - Keyspace keyspace = Keyspace.open(descriptor.ksname); - - // Make it works for indexes too - find parent cf if necessary - String baseName = descriptor.cfname; - if (descriptor.cfname.contains(".")) - { - String[] parts = descriptor.cfname.split("\\.", 2); - baseName = parts[0]; - } - - // IllegalArgumentException will be thrown here if ks/cf pair does not exist - ColumnFamilyStore cfStore = null; - try - { - cfStore = keyspace.getColumnFamilyStore(baseName); - } - catch (IllegalArgumentException e) - { - System.err.println(String.format("The provided table is not part of this cassandra keyspace: keyspace = %s, table = %s", - descriptor.ksname, descriptor.cfname)); - System.exit(1); - } - - try - { - if (cmd.hasOption(ENUMERATEKEYS_OPTION)) - { - enumeratekeys(descriptor, System.out, cfStore.metadata); - } - else - { - if ((keys != null) && (keys.length > 0)) - export(descriptor, System.out, Arrays.asList(keys), excludes, cfStore.metadata); - else - export(descriptor, excludes); - } - } - catch (IOException e) - { - // throwing exception outside main with broken pipe causes windows cmd to hang - e.printStackTrace(System.err); - } - - System.exit(0); - } - - private static void writeJSON(PrintStream out, Object value) - { - try - { - jsonMapper.writeValue(out, value); - } - catch (Exception e) - { - throw new RuntimeException(e.getMessage(), e); - } - } -} diff --git a/src/java/org/apache/cassandra/tools/SSTableImport.java b/src/java/org/apache/cassandra/tools/SSTableImport.java deleted file mode 100644 index b2d63aaf3a..0000000000 --- a/src/java/org/apache/cassandra/tools/SSTableImport.java +++ /dev/null @@ -1,568 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.apache.cassandra.tools; - -import java.io.File; -import java.io.IOException; -import java.nio.ByteBuffer; -import java.util.List; -import java.util.Map; -import java.util.SortedMap; -import java.util.TreeMap; -import java.util.concurrent.TimeUnit; - -import org.apache.cassandra.io.sstable.Descriptor; -import org.apache.cassandra.io.sstable.format.SSTableWriter; -import org.apache.commons.cli.CommandLine; -import org.apache.commons.cli.CommandLineParser; -import org.apache.commons.cli.Option; -import org.apache.commons.cli.Options; -import org.apache.commons.cli.ParseException; -import org.apache.commons.cli.PosixParser; - -import org.apache.cassandra.config.CFMetaData; -import org.apache.cassandra.db.*; -import org.apache.cassandra.db.composites.*; -import org.apache.cassandra.exceptions.ConfigurationException; -import org.apache.cassandra.config.DatabaseDescriptor; -import org.apache.cassandra.config.Schema; -import org.apache.cassandra.db.marshal.AbstractType; -import org.apache.cassandra.db.marshal.BytesType; -import org.apache.cassandra.dht.IPartitioner; -import org.apache.cassandra.serializers.MarshalException; -import org.apache.cassandra.service.ActiveRepairService; -import org.apache.cassandra.utils.ByteBufferUtil; -import org.apache.cassandra.utils.JVMStabilityInspector; -import org.codehaus.jackson.JsonFactory; -import org.codehaus.jackson.JsonParser; -import org.codehaus.jackson.JsonToken; -import org.codehaus.jackson.map.MappingJsonFactory; -import org.codehaus.jackson.type.TypeReference; - -/** - * Create SSTables from JSON input - */ -public class SSTableImport -{ - private static final String KEYSPACE_OPTION = "K"; - private static final String COLUMN_FAMILY_OPTION = "c"; - private static final String KEY_COUNT_OPTION = "n"; - private static final String IS_SORTED_OPTION = "s"; - - private static final Options options = new Options(); - private static CommandLine cmd; - - private Integer keyCountToImport; - private final boolean isSorted; - - private static final JsonFactory factory = new MappingJsonFactory().configure( - JsonParser.Feature.INTERN_FIELD_NAMES, false); - - static - { - Option optKeyspace = new Option(KEYSPACE_OPTION, true, "Keyspace name."); - optKeyspace.setRequired(true); - options.addOption(optKeyspace); - - Option optColfamily = new Option(COLUMN_FAMILY_OPTION, true, "Table name."); - optColfamily.setRequired(true); - options.addOption(optColfamily); - - options.addOption(new Option(KEY_COUNT_OPTION, true, "Number of keys to import (Optional).")); - options.addOption(new Option(IS_SORTED_OPTION, false, "Assume JSON file as already sorted (e.g. created by sstable2json tool) (Optional).")); - } - - private static class JsonColumn - { - private ByteBuffer name; - private ByteBuffer value; - private long timestamp; - - private String kind; - // Expiring columns - private int ttl; - private int localExpirationTime; - - // Counter columns - private long timestampOfLastDelete; - - public JsonColumn(T json, CFMetaData meta) - { - if (json instanceof List) - { - CellNameType comparator = meta.comparator; - List fields = (List) json; - - assert fields.size() >= 3 : "Cell definition should have at least 3"; - - name = stringAsType((String) fields.get(0), comparator.asAbstractType()); - timestamp = (Long) fields.get(2); - kind = ""; - - if (fields.size() > 3) - { - kind = (String) fields.get(3); - if (isExpiring()) - { - ttl = (Integer) fields.get(4); - localExpirationTime = (Integer) fields.get(5); - } - else if (isCounter()) - { - timestampOfLastDelete = ((Integer) fields.get(4)); - } - else if (isRangeTombstone()) - { - localExpirationTime = (Integer) fields.get(4); - } - } - - if (isDeleted()) - { - value = ByteBufferUtil.bytes((Integer) fields.get(1)); - } - else if (isRangeTombstone()) - { - value = stringAsType((String) fields.get(1), comparator.asAbstractType()); - } - else - { - assert meta.isCQL3Table() || name.hasRemaining() : "Cell name should not be empty"; - value = stringAsType((String) fields.get(1), - meta.getValueValidator(name.hasRemaining() - ? comparator.cellFromByteBuffer(name) - : meta.comparator.rowMarker(Composites.EMPTY))); - } - } - } - - public boolean isDeleted() - { - return kind.equals("d"); - } - - public boolean isExpiring() - { - return kind.equals("e"); - } - - public boolean isCounter() - { - return kind.equals("c"); - } - - public boolean isRangeTombstone() - { - return kind.equals("t"); - } - - public ByteBuffer getName() - { - return name.duplicate(); - } - - public ByteBuffer getValue() - { - return value.duplicate(); - } - } - - public SSTableImport() - { - this(null, false); - } - - public SSTableImport(boolean isSorted) - { - this(null, isSorted); - } - - public SSTableImport(Integer keyCountToImport, boolean isSorted) - { - this.keyCountToImport = keyCountToImport; - this.isSorted = isSorted; - } - - /** - * Add columns to a column family. - * - * @param row the columns associated with a row - * @param cfamily the column family to add columns to - */ - private void addColumnsToCF(List row, ColumnFamily cfamily) - { - CFMetaData cfm = cfamily.metadata(); - assert cfm != null; - - for (Object c : row) - { - JsonColumn col = new JsonColumn((List) c, cfm); - if (col.isRangeTombstone()) - { - Composite start = cfm.comparator.fromByteBuffer(col.getName()); - Composite end = cfm.comparator.fromByteBuffer(col.getValue()); - cfamily.addAtom(new RangeTombstone(start, end, col.timestamp, col.localExpirationTime)); - continue; - } - - assert cfm.isCQL3Table() || col.getName().hasRemaining() : "Cell name should not be empty"; - CellName cname = col.getName().hasRemaining() ? cfm.comparator.cellFromByteBuffer(col.getName()) - : cfm.comparator.rowMarker(Composites.EMPTY); - - if (col.isExpiring()) - { - cfamily.addColumn(new BufferExpiringCell(cname, col.getValue(), col.timestamp, col.ttl, col.localExpirationTime)); - } - else if (col.isCounter()) - { - cfamily.addColumn(new BufferCounterCell(cname, col.getValue(), col.timestamp, col.timestampOfLastDelete)); - } - else if (col.isDeleted()) - { - cfamily.addTombstone(cname, col.getValue(), col.timestamp); - } - else if (col.isRangeTombstone()) - { - CellName end = cfm.comparator.cellFromByteBuffer(col.getValue()); - cfamily.addAtom(new RangeTombstone(cname, end, col.timestamp, col.localExpirationTime)); - } - // cql3 row marker, see CASSANDRA-5852 - else if (cname.isEmpty()) - { - cfamily.addColumn(cfm.comparator.rowMarker(Composites.EMPTY), col.getValue(), col.timestamp); - } - else - { - cfamily.addColumn(cname, col.getValue(), col.timestamp); - } - } - } - - private void parseMeta(Map map, ColumnFamily cf, ByteBuffer superColumnName) - { - - // deletionInfo is the only metadata we store for now - if (map.containsKey("deletionInfo")) - { - Map unparsedDeletionInfo = (Map) map.get("deletionInfo"); - Number number = (Number) unparsedDeletionInfo.get("markedForDeleteAt"); - long markedForDeleteAt = number instanceof Long ? (Long) number : number.longValue(); - int localDeletionTime = (Integer) unparsedDeletionInfo.get("localDeletionTime"); - if (superColumnName == null) - cf.setDeletionInfo(new DeletionInfo(markedForDeleteAt, localDeletionTime)); - else - cf.addAtom(new RangeTombstone(SuperColumns.startOf(superColumnName), SuperColumns.endOf(superColumnName), markedForDeleteAt, localDeletionTime)); - } - } - - /** - * Convert a JSON formatted file to an SSTable. - * - * @param jsonFile the file containing JSON formatted data - * @param keyspace keyspace the data belongs to - * @param cf column family the data belongs to - * @param ssTablePath file to write the SSTable to - * - * @throws IOException for errors reading/writing input/output - */ - public int importJson(String jsonFile, String keyspace, String cf, String ssTablePath) throws IOException - { - ColumnFamily columnFamily = ArrayBackedSortedColumns.factory.create(keyspace, cf); - IPartitioner partitioner = DatabaseDescriptor.getPartitioner(); - - int importedKeys = (isSorted) ? importSorted(jsonFile, columnFamily, ssTablePath, partitioner) - : importUnsorted(jsonFile, columnFamily, ssTablePath, partitioner); - - if (importedKeys != -1) - System.out.printf("%d keys imported successfully.%n", importedKeys); - - return importedKeys; - } - - private int importUnsorted(String jsonFile, ColumnFamily columnFamily, String ssTablePath, IPartitioner partitioner) throws IOException - { - int importedKeys = 0; - long start = System.nanoTime(); - - Object[] data; - try (JsonParser parser = getParser(jsonFile)) - { - data = parser.readValueAs(new TypeReference(){}); - } - - keyCountToImport = (keyCountToImport == null) ? data.length : keyCountToImport; - - try (SSTableWriter writer = SSTableWriter.create(Descriptor.fromFilename(ssTablePath), keyCountToImport, ActiveRepairService.UNREPAIRED_SSTABLE, 0)) - { - System.out.printf("Importing %s keys...%n", keyCountToImport); - - // sort by dk representation, but hold onto the hex version - SortedMap> decoratedKeys = new TreeMap>(); - - for (Object row : data) - { - Map rowAsMap = (Map) row; - decoratedKeys.put(partitioner.decorateKey(getKeyValidator(columnFamily).fromString((String) rowAsMap.get("key"))), rowAsMap); - } - - for (Map.Entry> row : decoratedKeys.entrySet()) - { - if (row.getValue().containsKey("metadata")) - { - parseMeta((Map) row.getValue().get("metadata"), columnFamily, null); - } - - Object columns = row.getValue().get("cells"); - addColumnsToCF((List) columns, columnFamily); - - - writer.append(row.getKey(), columnFamily); - columnFamily.clear(); - - importedKeys++; - - long current = System.nanoTime(); - - if (TimeUnit.NANOSECONDS.toSeconds(current - start) >= 5) // 5 secs. - { - System.out.printf("Currently imported %d keys.%n", importedKeys); - start = current; - } - - if (keyCountToImport == importedKeys) - break; - } - - writer.finish(true); - } - - return importedKeys; - } - - private int importSorted(String jsonFile, ColumnFamily columnFamily, String ssTablePath, - IPartitioner partitioner) throws IOException - { - int importedKeys = 0; // already imported keys count - long start = System.nanoTime(); - - try (JsonParser parser = getParser(jsonFile)) - { - - if (keyCountToImport == null) - { - keyCountToImport = 0; - System.out.println("Counting keys to import, please wait... (NOTE: to skip this use -n )"); - - parser.nextToken(); // START_ARRAY - while (parser.nextToken() != null) - { - parser.skipChildren(); - if (parser.getCurrentToken() == JsonToken.END_ARRAY) - break; - - keyCountToImport++; - } - } - System.out.printf("Importing %s keys...%n", keyCountToImport); - } - - try (JsonParser parser = getParser(jsonFile); // renewing parser - SSTableWriter writer = SSTableWriter.create(Descriptor.fromFilename(ssTablePath), keyCountToImport, ActiveRepairService.UNREPAIRED_SSTABLE);) - { - int lineNumber = 1; - DecoratedKey prevStoredKey = null; - - parser.nextToken(); // START_ARRAY - while (parser.nextToken() != null) - { - String key = parser.getCurrentName(); - Map row = parser.readValueAs(new TypeReference>(){}); - DecoratedKey currentKey = partitioner.decorateKey(getKeyValidator(columnFamily).fromString((String) row.get("key"))); - - if (row.containsKey("metadata")) - parseMeta((Map) row.get("metadata"), columnFamily, null); - - addColumnsToCF((List) row.get("cells"), columnFamily); - - if (prevStoredKey != null && prevStoredKey.compareTo(currentKey) != -1) - { - System.err - .printf("Line %d: Key %s is greater than previous, collection is not sorted properly. Aborting import. You might need to delete SSTables manually.%n", - lineNumber, key); - return -1; - } - - // saving decorated key - writer.append(currentKey, columnFamily); - columnFamily.clear(); - - prevStoredKey = currentKey; - importedKeys++; - lineNumber++; - - long current = System.nanoTime(); - - if (TimeUnit.NANOSECONDS.toSeconds(current - start) >= 5) // 5 secs. - { - System.out.printf("Currently imported %d keys.%n", importedKeys); - start = current; - } - - if (keyCountToImport == importedKeys) - break; - - } - - writer.finish(true); - - return importedKeys; - } - } - - /** - * Get key validator for column family - * @param columnFamily column family instance - * @return key validator for given column family - */ - private AbstractType getKeyValidator(ColumnFamily columnFamily) { - // this is a fix to support backward compatibility - // which allows to skip the current key validator - // please, take a look onto CASSANDRA-7498 for more details - if ("true".equals(System.getProperty("skip.key.validator", "false"))) { - return BytesType.instance; - } - return columnFamily.metadata().getKeyValidator(); - } - - /** - * Get JsonParser object for file - * @param fileName name of the file - * @return json parser instance for given file - * @throws IOException if any I/O error. - */ - private JsonParser getParser(String fileName) throws IOException - { - return factory.createJsonParser(new File(fileName)); - } - - /** - * Converts JSON to an SSTable file. JSON input can either be a file specified - * using an optional command line argument, or supplied on standard in. - * - * @param args command line arguments - * @throws ParseException on failure to parse JSON input - * @throws ConfigurationException on configuration error. - */ - public static void main(String[] args) throws ParseException, ConfigurationException - { - System.err.println("WARNING: please note that json2sstable is now deprecated and will be removed in Cassandra 3.0. " - + "You should use CQLSSTableWriter if you want to write sstables directly. " - + "Please see https://issues.apache.org/jira/browse/CASSANDRA-9618 for details."); - - CommandLineParser parser = new PosixParser(); - - try - { - cmd = parser.parse(options, args); - } - catch (org.apache.commons.cli.ParseException e) - { - System.err.println(e.getMessage()); - printProgramUsage(); - System.exit(1); - } - - if (cmd.getArgs().length != 2) - { - printProgramUsage(); - System.exit(1); - } - - String json = cmd.getArgs()[0]; - String ssTable = cmd.getArgs()[1]; - String keyspace = cmd.getOptionValue(KEYSPACE_OPTION); - String cfamily = cmd.getOptionValue(COLUMN_FAMILY_OPTION); - - Integer keyCountToImport = null; - boolean isSorted = false; - - if (cmd.hasOption(KEY_COUNT_OPTION)) - { - keyCountToImport = Integer.valueOf(cmd.getOptionValue(KEY_COUNT_OPTION)); - } - - if (cmd.hasOption(IS_SORTED_OPTION)) - { - isSorted = true; - } - - Schema.instance.loadFromDisk(false); - if (Schema.instance.getNonSystemKeyspaces().size() < 1) - { - String msg = "no non-system keyspaces are defined"; - System.err.println(msg); - throw new ConfigurationException(msg); - } - - try - { - new SSTableImport(keyCountToImport, isSorted).importJson(json, keyspace, cfamily, ssTable); - } - catch (Exception e) - { - JVMStabilityInspector.inspectThrowable(e); - e.printStackTrace(); - System.err.println("ERROR: " + e.getMessage()); - System.exit(-1); - } - - System.exit(0); - } - - private static void printProgramUsage() - { - System.out.printf("Usage: %s -s -K -c -n %n%n", - SSTableImport.class.getName()); - - System.out.println("Options:"); - for (Object o : options.getOptions()) - { - Option opt = (Option) o; - System.out.println(" -" +opt.getOpt() + " - " + opt.getDescription()); - } - } - - /** - * Convert a string to bytes (ByteBuffer) according to type - * @param content string to convert - * @param type type to use for conversion - * @return byte buffer representation of the given string - */ - private static ByteBuffer stringAsType(String content, AbstractType type) - { - try - { - return type.fromString(content); - } - catch (MarshalException e) - { - throw new RuntimeException(e.getMessage()); - } - } - -} diff --git a/src/java/org/apache/cassandra/tracing/TraceKeyspace.java b/src/java/org/apache/cassandra/tracing/TraceKeyspace.java index f66269d06c..f4d645031f 100644 --- a/src/java/org/apache/cassandra/tracing/TraceKeyspace.java +++ b/src/java/org/apache/cassandra/tracing/TraceKeyspace.java @@ -25,9 +25,8 @@ import com.google.common.collect.ImmutableMap; import org.apache.cassandra.config.CFMetaData; import org.apache.cassandra.config.KSMetaData; -import org.apache.cassandra.db.CFRowAdder; -import org.apache.cassandra.db.ColumnFamily; import org.apache.cassandra.db.Mutation; +import org.apache.cassandra.db.RowUpdateBuilder; import org.apache.cassandra.locator.SimpleStrategy; import org.apache.cassandra.utils.FBUtilities; import org.apache.cassandra.utils.UUIDGen; @@ -85,44 +84,36 @@ public final class TraceKeyspace String command, int ttl) { - Mutation mutation = new Mutation(NAME, sessionId); - ColumnFamily cells = mutation.addOrGet(TraceKeyspace.Sessions); + RowUpdateBuilder adder = new RowUpdateBuilder(Sessions, FBUtilities.timestampMicros(), ttl, sessionId) + .clustering() + .add("client", client) + .add("coordinator", FBUtilities.getBroadcastAddress()) + .add("request", request) + .add("started_at", new Date(startedAt)) + .add("command", command); - CFRowAdder adder = new CFRowAdder(cells, cells.metadata().comparator.builder().build(), FBUtilities.timestampMicros(), ttl); - adder.add("client", client) - .add("coordinator", FBUtilities.getBroadcastAddress()) - .add("request", request) - .add("started_at", new Date(startedAt)) - .add("command", command); for (Map.Entry entry : parameters.entrySet()) adder.addMapEntry("parameters", entry.getKey(), entry.getValue()); - - return mutation; + return adder.build(); } static Mutation makeStopSessionMutation(ByteBuffer sessionId, int elapsed, int ttl) { - Mutation mutation = new Mutation(NAME, sessionId); - ColumnFamily cells = mutation.addOrGet(Sessions); - - CFRowAdder adder = new CFRowAdder(cells, cells.metadata().comparator.builder().build(), FBUtilities.timestampMicros(), ttl); - adder.add("duration", elapsed); - - return mutation; + return new RowUpdateBuilder(Sessions, FBUtilities.timestampMicros(), ttl, sessionId) + .clustering() + .add("duration", elapsed) + .build(); } static Mutation makeEventMutation(ByteBuffer sessionId, String message, int elapsed, String threadName, int ttl) { - Mutation mutation = new Mutation(NAME, sessionId); - ColumnFamily cells = mutation.addOrGet(Events); - - CFRowAdder adder = new CFRowAdder(cells, cells.metadata().comparator.make(UUIDGen.getTimeUUID()), FBUtilities.timestampMicros(), ttl); - adder.add("activity", message) - .add("source", FBUtilities.getBroadcastAddress()) - .add("thread", threadName); + RowUpdateBuilder adder = new RowUpdateBuilder(Events, FBUtilities.timestampMicros(), ttl, sessionId) + .clustering(UUIDGen.getTimeUUID()); + adder.add("activity", message); + adder.add("source", FBUtilities.getBroadcastAddress()); + adder.add("thread", threadName); if (elapsed >= 0) adder.add("source_elapsed", elapsed); - - return mutation; + return adder.build(); } } diff --git a/src/java/org/apache/cassandra/transport/messages/QueryMessage.java b/src/java/org/apache/cassandra/transport/messages/QueryMessage.java index 4e21678376..4e54e46fbe 100644 --- a/src/java/org/apache/cassandra/transport/messages/QueryMessage.java +++ b/src/java/org/apache/cassandra/transport/messages/QueryMessage.java @@ -141,6 +141,6 @@ public class QueryMessage extends Message.Request @Override public String toString() { - return "QUERY " + query; + return "QUERY " + query + "[pageSize = " + options.getPageSize() + "]"; } } diff --git a/src/java/org/apache/cassandra/triggers/ITrigger.java b/src/java/org/apache/cassandra/triggers/ITrigger.java index 21aba05daf..ad631d144d 100644 --- a/src/java/org/apache/cassandra/triggers/ITrigger.java +++ b/src/java/org/apache/cassandra/triggers/ITrigger.java @@ -24,11 +24,11 @@ package org.apache.cassandra.triggers; import java.nio.ByteBuffer; import java.util.Collection; -import org.apache.cassandra.db.ColumnFamily; import org.apache.cassandra.db.Mutation; +import org.apache.cassandra.db.partitions.Partition; /** - * Trigger interface, For every Mutation received by the coordinator {@link #augment(ByteBuffer, ColumnFamily)} + * Trigger interface, For every partition update received by the coordinator {@link #augment(Partition)} * is called.

* * Contract:
@@ -44,9 +44,8 @@ public interface ITrigger /** * Called exactly once per CF update, returned mutations are atomically updated. * - * @param partitionKey - partition Key for the update. * @param update - update received for the CF * @return additional modifications to be applied along with the supplied update */ - public Collection augment(ByteBuffer partitionKey, ColumnFamily update); + public Collection augment(Partition update); } diff --git a/src/java/org/apache/cassandra/triggers/TriggerExecutor.java b/src/java/org/apache/cassandra/triggers/TriggerExecutor.java index 973ad8b926..071a973ea6 100644 --- a/src/java/org/apache/cassandra/triggers/TriggerExecutor.java +++ b/src/java/org/apache/cassandra/triggers/TriggerExecutor.java @@ -22,13 +22,16 @@ import java.io.File; import java.nio.ByteBuffer; import java.util.*; +import com.google.common.collect.ArrayListMultimap; import com.google.common.collect.Iterables; import com.google.common.collect.Lists; +import com.google.common.collect.ListMultimap; import com.google.common.collect.Maps; import org.apache.cassandra.config.TriggerDefinition; import org.apache.cassandra.cql3.QueryProcessor; import org.apache.cassandra.db.*; +import org.apache.cassandra.db.partitions.PartitionUpdate; import org.apache.cassandra.db.marshal.AbstractType; import org.apache.cassandra.exceptions.InvalidRequestException; import org.apache.cassandra.utils.FBUtilities; @@ -62,7 +65,7 @@ public class TriggerExecutor /** * Augment a partition update by executing triggers to generate an intermediate - * set of mutations, then merging the ColumnFamily from each mutation with those + * set of mutations, then merging the update from each mutation with those * supplied. This is called from @{link org.apache.cassandra.service.StorageProxy#cas} * which is scoped for a single partition. For that reason, any mutations generated * by triggers are checked to ensure that they are for the same table and partition @@ -77,22 +80,13 @@ public class TriggerExecutor * @throws InvalidRequestException if any mutation generated by a trigger does not * apply to the exact same partition as the initial update */ - public ColumnFamily execute(ByteBuffer key, ColumnFamily updates) throws InvalidRequestException + public PartitionUpdate execute(PartitionUpdate updates) throws InvalidRequestException { - List intermediate = executeInternal(key, updates); + List intermediate = executeInternal(updates); if (intermediate == null || intermediate.isEmpty()) return updates; - validateForSinglePartition(updates.metadata().getKeyValidator(), updates.id(), key, intermediate); - - for (Mutation mutation : intermediate) - { - for (ColumnFamily cf : mutation.getColumnFamilies()) - { - updates.addAll(cf); - } - } - return updates; + return PartitionUpdate.merge(validateForSinglePartition(updates.metadata().cfId, updates.partitionKey(), intermediate)); } /** @@ -120,9 +114,9 @@ public class TriggerExecutor if (mutation instanceof CounterMutation) hasCounters = true; - for (ColumnFamily cf : mutation.getColumnFamilies()) + for (PartitionUpdate upd : mutation.getPartitionUpdates()) { - List augmentations = executeInternal(mutation.key(), cf); + List augmentations = executeInternal(upd); if (augmentations == null || augmentations.isEmpty()) continue; @@ -148,54 +142,66 @@ public class TriggerExecutor private Collection mergeMutations(Iterable mutations) { - Map, Mutation> groupedMutations = new HashMap<>(); + ListMultimap, Mutation> groupedMutations = ArrayListMultimap.create(); for (Mutation mutation : mutations) { - Pair key = Pair.create(mutation.getKeyspaceName(), mutation.key()); - Mutation current = groupedMutations.get(key); - if (current == null) - { - // copy in case the mutation's modifications map is backed by an immutable Collections#singletonMap(). - groupedMutations.put(key, mutation.copy()); - } - else - { - current.addAll(mutation); - } + Pair key = Pair.create(mutation.getKeyspaceName(), mutation.key().getKey()); + groupedMutations.put(key, mutation); } - return groupedMutations.values(); + List merged = new ArrayList<>(groupedMutations.size()); + for (Pair key : groupedMutations.keySet()) + merged.add(Mutation.merge(groupedMutations.get(key))); + + return merged; } - private void validateForSinglePartition(AbstractType keyValidator, - UUID cfId, - ByteBuffer key, - Collection tmutations) + private Collection validateForSinglePartition(UUID cfId, + DecoratedKey key, + Collection tmutations) throws InvalidRequestException { + validate(tmutations); + + if (tmutations.size() == 1) + { + Collection updates = Iterables.getOnlyElement(tmutations).getPartitionUpdates(); + if (updates.size() > 1) + throw new InvalidRequestException("The updates generated by triggers are not all for the same partition"); + validateSamePartition(cfId, key, Iterables.getOnlyElement(updates)); + return updates; + } + + ArrayList updates = new ArrayList<>(tmutations.size()); for (Mutation mutation : tmutations) { - if (keyValidator.compare(mutation.key(), key) != 0) - throw new InvalidRequestException("Partition key of additional mutation does not match primary update key"); - - for (ColumnFamily cf : mutation.getColumnFamilies()) + for (PartitionUpdate update : mutation.getPartitionUpdates()) { - if (! cf.id().equals(cfId)) - throw new InvalidRequestException("table of additional mutation does not match primary update table"); + validateSamePartition(cfId, key, update); + updates.add(update); } } - validate(tmutations); + return updates; + } + + private void validateSamePartition(UUID cfId, DecoratedKey key, PartitionUpdate update) + throws InvalidRequestException + { + if (!key.equals(update.partitionKey())) + throw new InvalidRequestException("Partition key of additional mutation does not match primary update key"); + + if (!cfId.equals(update.metadata().cfId)) + throw new InvalidRequestException("table of additional mutation does not match primary update table"); } private void validate(Collection tmutations) throws InvalidRequestException { for (Mutation mutation : tmutations) { - QueryProcessor.validateKey(mutation.key()); - for (ColumnFamily tcf : mutation.getColumnFamilies()) - for (Cell cell : tcf) - cell.validateFields(tcf.metadata()); + QueryProcessor.validateKey(mutation.key().getKey()); + for (PartitionUpdate update : mutation.getPartitionUpdates()) + update.validate(); } } @@ -203,9 +209,9 @@ public class TriggerExecutor * Switch class loader before using the triggers for the column family, if * not loaded them with the custom class loader. */ - private List executeInternal(ByteBuffer key, ColumnFamily columnFamily) + private List executeInternal(PartitionUpdate update) { - Map triggers = columnFamily.metadata().getTriggers(); + Map triggers = update.metadata().getTriggers(); if (triggers.isEmpty()) return null; List tmutations = Lists.newLinkedList(); @@ -220,7 +226,7 @@ public class TriggerExecutor trigger = loadTriggerInstance(td.classOption); cachedTriggers.put(td.classOption, trigger); } - Collection temp = trigger.augment(key, columnFamily); + Collection temp = trigger.augment(update); if (temp != null) tmutations.addAll(temp); } @@ -228,7 +234,7 @@ public class TriggerExecutor } catch (Exception ex) { - throw new RuntimeException(String.format("Exception while creating trigger on table with ID: %s", columnFamily.id()), ex); + throw new RuntimeException(String.format("Exception while creating trigger on table with ID: %s", update.metadata().cfId), ex); } finally { diff --git a/src/java/org/apache/cassandra/utils/ByteBufferUtil.java b/src/java/org/apache/cassandra/utils/ByteBufferUtil.java index 1831c19031..69915bf3ba 100644 --- a/src/java/org/apache/cassandra/utils/ByteBufferUtil.java +++ b/src/java/org/apache/cassandra/utils/ByteBufferUtil.java @@ -33,6 +33,7 @@ import java.util.Arrays; import java.util.UUID; import net.nicoulaj.compilecommand.annotations.Inline; +import org.apache.cassandra.db.TypeSizes; import org.apache.cassandra.io.util.DataOutputPlus; import org.apache.cassandra.io.util.FileDataInput; import org.apache.cassandra.io.util.FileUtils; @@ -322,6 +323,12 @@ public class ByteBufferUtil return ByteBufferUtil.read(in, length); } + public static int serializedSizeWithLength(ByteBuffer buffer, TypeSizes sizes) + { + int size = buffer.remaining(); + return sizes.sizeof(size) + size; + } + /* @return An unsigned short in an integer. */ public static int readShortLength(DataInput in) throws IOException { @@ -338,16 +345,21 @@ public class ByteBufferUtil return ByteBufferUtil.read(in, readShortLength(in)); } + public static int serializedSizeWithShortLength(ByteBuffer buffer, TypeSizes sizes) + { + int size = buffer.remaining(); + return sizes.sizeof((short)size) + size; + } + /** * @param in data input * @return null * @throws IOException if an I/O error occurs. */ - public static ByteBuffer skipShortLength(DataInput in) throws IOException + public static void skipShortLength(DataInput in) throws IOException { int skip = readShortLength(in); FileUtils.skipBytesFully(in, skip); - return null; } public static ByteBuffer read(DataInput in, int length) throws IOException diff --git a/src/java/org/apache/cassandra/utils/FBUtilities.java b/src/java/org/apache/cassandra/utils/FBUtilities.java index 17edeb0d10..b0bed7b80c 100644 --- a/src/java/org/apache/cassandra/utils/FBUtilities.java +++ b/src/java/org/apache/cassandra/utils/FBUtilities.java @@ -372,6 +372,11 @@ public class FBUtilities return System.currentTimeMillis() * 1000; } + public static int nowInSeconds() + { + return (int)(System.currentTimeMillis() / 1000); + } + public static void waitOnFutures(Iterable> futures) { for (Future f : futures) @@ -502,13 +507,18 @@ public class FBUtilities } } - public static SortedSet singleton(T column, Comparator comparator) + public static NavigableSet singleton(T column, Comparator comparator) { - SortedSet s = new TreeSet(comparator); + NavigableSet s = new TreeSet(comparator); s.add(column); return s; } + public static NavigableSet emptySortedSet(Comparator comparator) + { + return new TreeSet(comparator); + } + public static String toString(Map map) { Joiner.MapJoiner joiner = Joiner.on(", ").withKeyValueSeparator(":"); @@ -796,4 +806,30 @@ public class FBUtilities digest.update((byte) ((val >>> 8) & 0xFF)); digest.update((byte) ((val >>> 0) & 0xFF)); } + + public static void updateWithBoolean(MessageDigest digest, boolean val) + { + updateWithByte(digest, val ? 0 : 1); + } + + public static void closeAll(List l) throws Exception + { + Exception toThrow = null; + for (AutoCloseable c : l) + { + try + { + c.close(); + } + catch (Exception e) + { + if (toThrow == null) + toThrow = e; + else + toThrow.addSuppressed(e); + } + } + if (toThrow != null) + throw toThrow; + } } diff --git a/src/java/org/apache/cassandra/utils/MergeIterator.java b/src/java/org/apache/cassandra/utils/MergeIterator.java index e61326e2a9..d0f116ed3e 100644 --- a/src/java/org/apache/cassandra/utils/MergeIterator.java +++ b/src/java/org/apache/cassandra/utils/MergeIterator.java @@ -18,7 +18,6 @@ package org.apache.cassandra.utils; import java.io.Closeable; -import java.io.IOException; import java.util.*; import com.google.common.collect.AbstractIterator; @@ -35,9 +34,9 @@ public abstract class MergeIterator extends AbstractIterator implem this.reducer = reducer; } - public static IMergeIterator get(List> sources, - Comparator comparator, - Reducer reducer) + public static MergeIterator get(List> sources, + Comparator comparator, + Reducer reducer) { if (sources.size() == 1) { @@ -59,9 +58,10 @@ public abstract class MergeIterator extends AbstractIterator implem { try { - ((Closeable)iterator).close(); + if (iterator instanceof AutoCloseable) + ((AutoCloseable)iterator).close(); } - catch (IOException e) + catch (Exception e) { throw new RuntimeException(e); } @@ -79,13 +79,13 @@ public abstract class MergeIterator extends AbstractIterator implem // TODO: if we had our own PriorityQueue implementation we could stash items // at the end of its array, so we wouldn't need this storage protected final ArrayDeque> candidates; - public ManyToOne(List> iters, Comparator comp, Reducer reducer) + public ManyToOne(List> iters, Comparator comp, Reducer reducer) { super(iters, reducer); this.queue = new PriorityQueue<>(Math.max(1, iters.size())); - for (Iterator iter : iters) + for (int i = 0; i < iters.size(); i++) { - Candidate candidate = new Candidate<>(iter, comp); + Candidate candidate = new Candidate<>(i, iters.get(i), comp); if (!candidate.advance()) // was empty continue; @@ -111,7 +111,7 @@ public abstract class MergeIterator extends AbstractIterator implem { candidate = queue.poll(); candidates.push(candidate); - reducer.reduce(candidate.item); + reducer.reduce(candidate.idx, candidate.item); } while (queue.peek() != null && queue.peek().compareTo(candidate) == 0); return reducer.getReduced(); @@ -130,14 +130,16 @@ public abstract class MergeIterator extends AbstractIterator implem // Holds and is comparable by the head item of an iterator it owns protected static final class Candidate implements Comparable> { - private final Iterator iter; - private final Comparator comp; + private final Iterator iter; + private final Comparator comp; + private final int idx; private In item; - public Candidate(Iterator iter, Comparator comp) + public Candidate(int idx, Iterator iter, Comparator comp) { this.iter = iter; this.comp = comp; + this.idx = idx; } /** @return True if our iterator had an item, and it is now available */ @@ -170,7 +172,7 @@ public abstract class MergeIterator extends AbstractIterator implem * combine this object with the previous ones. * intermediate state is up to your implementation. */ - public abstract void reduce(In current); + public abstract void reduce(int idx, In current); /** @return The last object computed by reduce */ protected abstract Out getReduced(); @@ -202,7 +204,7 @@ public abstract class MergeIterator extends AbstractIterator implem if (!source.hasNext()) return endOfData(); reducer.onKeyChange(); - reducer.reduce(source.next()); + reducer.reduce(0, source.next()); return reducer.getReduced(); } } diff --git a/src/java/org/apache/cassandra/utils/NativeSSTableLoaderClient.java b/src/java/org/apache/cassandra/utils/NativeSSTableLoaderClient.java index 5448390ad4..f2663bf4ef 100644 --- a/src/java/org/apache/cassandra/utils/NativeSSTableLoaderClient.java +++ b/src/java/org/apache/cassandra/utils/NativeSSTableLoaderClient.java @@ -21,13 +21,13 @@ import java.net.InetAddress; import java.util.*; import com.datastax.driver.core.*; + +import org.apache.cassandra.config.ColumnDefinition; import org.apache.cassandra.config.CFMetaData; -import org.apache.cassandra.db.ColumnFamilyType; +import org.apache.cassandra.cql3.ColumnIdentifier; +import org.apache.cassandra.db.CompactTables; import org.apache.cassandra.db.SystemKeyspace; -import org.apache.cassandra.db.composites.CellNameType; -import org.apache.cassandra.db.composites.CellNames; -import org.apache.cassandra.db.marshal.AbstractType; -import org.apache.cassandra.db.marshal.TypeParser; +import org.apache.cassandra.db.marshal.*; import org.apache.cassandra.dht.*; import org.apache.cassandra.dht.Token; import org.apache.cassandra.io.sstable.SSTableLoader; @@ -100,27 +100,70 @@ public class NativeSSTableLoaderClient extends SSTableLoader.Client { Map tables = new HashMap<>(); - String query = String.format("SELECT columnfamily_name, cf_id, type, comparator, subcomparator, is_dense FROM %s.%s WHERE keyspace_name = '%s'", + String query = String.format("SELECT columnfamily_name, cf_id, type, comparator, subcomparator, is_dense, default_validator FROM %s.%s WHERE keyspace_name = '%s'", SystemKeyspace.NAME, LegacySchemaTables.COLUMNFAMILIES, keyspace); + + // The following is a slightly simplified but otherwise duplicated version of LegacySchemaTables.createTableFromTableRowAndColumnRows. It might + // be safer to have a simple wrapper of the driver ResultSet/Row implementing UntypedResultSet/UntypedResultSet.Row and reuse the original method. for (Row row : session.execute(query)) { String name = row.getString("columnfamily_name"); UUID id = row.getUUID("cf_id"); - ColumnFamilyType type = ColumnFamilyType.valueOf(row.getString("type")); + boolean isSuper = row.getString("type").toLowerCase().equals("super"); AbstractType rawComparator = TypeParser.parse(row.getString("comparator")); AbstractType subComparator = row.isNull("subcomparator") ? null : TypeParser.parse(row.getString("subcomparator")); boolean isDense = row.getBool("is_dense"); - CellNameType comparator = CellNames.fromAbstractType(CFMetaData.makeRawAbstractType(rawComparator, subComparator), - isDense); + boolean isCompound = rawComparator instanceof CompositeType; - tables.put(name, new CFMetaData(keyspace, name, type, comparator, id)); + AbstractType defaultValidator = TypeParser.parse(row.getString("default_validator")); + boolean isCounter = defaultValidator instanceof CounterColumnType; + boolean isCQLTable = !isSuper && !isDense && isCompound; + + String columnsQuery = String.format("SELECT column_name, component_index, type, validator FROM %s.%s WHERE keyspace_name='%s' AND columnfamily_name='%s'", + SystemKeyspace.NAME, + LegacySchemaTables.COLUMNS, + keyspace, + name); + + List defs = new ArrayList<>(); + for (Row colRow : session.execute(columnsQuery)) + defs.add(createDefinitionFromRow(colRow, keyspace, name, rawComparator, subComparator, isSuper, isCQLTable)); + + tables.put(name, CFMetaData.create(keyspace, name, id, isDense, isCompound, isSuper, isCounter, defs)); } return tables; } + + // A slightly simplified version of LegacySchemaTables. + private static ColumnDefinition createDefinitionFromRow(Row row, + String keyspace, + String table, + AbstractType rawComparator, + AbstractType rawSubComparator, + boolean isSuper, + boolean isCQLTable) + { + ColumnDefinition.Kind kind = LegacySchemaTables.deserializeKind(row.getString("type")); + + Integer componentIndex = null; + if (!row.isNull("component_index")) + componentIndex = row.getInt("component_index"); + + // Note: we save the column name as string, but we should not assume that it is an UTF8 name, we + // we need to use the comparator fromString method + AbstractType comparator = isCQLTable + ? UTF8Type.instance + : CompactTables.columnDefinitionComparator(kind, isSuper, rawComparator, rawSubComparator); + ColumnIdentifier name = ColumnIdentifier.getInterned(comparator.fromString(row.getString("column_name")), comparator); + + AbstractType validator = TypeParser.parse(row.getString("validator")); + + return new ColumnDefinition(keyspace, table, name, validator, null, null, null, componentIndex, kind); + } } diff --git a/src/java/org/apache/cassandra/utils/Sorting.java b/src/java/org/apache/cassandra/utils/Sorting.java new file mode 100644 index 0000000000..b1c0b46c2a --- /dev/null +++ b/src/java/org/apache/cassandra/utils/Sorting.java @@ -0,0 +1,254 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.cassandra.utils; + +public abstract class Sorting +{ + private Sorting() {} + + /** + * Interface that allows to sort elements addressable by index, but without actually requiring those + * to elements to be part of a list/array. + */ + public interface Sortable + { + /** + * The number of elements to sort. + */ + public int size(); + + /** + * Compares the element with index i should sort before the element with index j. + */ + public int compare(int i, int j); + + /** + * Swaps element i and j. + */ + public void swap(int i, int j); + } + + /** + * Sort a sortable. + * + * The actual algorithm is a direct adaptation of the standard sorting in golang + * at http://golang.org/src/pkg/sort/sort.go (comments included). + * + * It makes one call to data.Len to determine n, and O(n*log(n)) calls to + * data.Less and data.Swap. The sort is not guaranteed to be stable. + */ + public static void sort(Sortable data) + { + // Switch to heapsort if depth of 2*ceil(lg(n+1)) is reached. + int n = data.size(); + int maxDepth = 0; + for (int i = n; i > 0; i >>= 1) + maxDepth++; + maxDepth *= 2; + quickSort(data, 0, n, maxDepth); + } + + private static void insertionSort(Sortable data, int a, int b) + { + for (int i = a + 1; i < b; i++) + for(int j = i; j > a && data.compare(j, j-1) < 0; j--) + data.swap(j, j-1); + } + + // siftDown implements the heap property on data[lo, hi). + // first is an offset into the array where the root of the heap lies. + private static void siftDown(Sortable data, int lo, int hi, int first) + { + int root = lo; + while (true) + { + int child = 2*root + 1; + if (child >= hi) + return; + + if (child + 1 < hi && data.compare(first+child, first+child+1) < 0) + child++; + + if (data.compare(first+root, first+child) >= 0) + return; + + data.swap(first+root, first+child); + root = child; + } + } + + private static void heapSort(Sortable data, int a, int b) + { + int first = a; + int lo = 0; + int hi = b - a; + + // Build heap with greatest element at top. + for (int i = (hi - 1) / 2; i >= 0; i--) + siftDown(data, i, hi, first); + + // Pop elements, largest first, into end of data. + for (int i = hi - 1; i >= 0; i--) { + data.swap(first, first+i); + siftDown(data, lo, i, first); + } + } + + // Quicksort, following Bentley and McIlroy, + // ``Engineering a Sort Function,'' SP&E November 1993. + + // medianOfThree moves the median of the three values data[a], data[b], data[c] into data[a]. + private static void medianOfThree(Sortable data, int a, int b, int c) + { + int m0 = b; + int m1 = a; + int m2 = c; + // bubble sort on 3 elements + if (data.compare(m1, m0) < 0) + data.swap(m1, m0); + if (data.compare(m2, m1) < 0) + data.swap(m2, m1); + if (data.compare(m1, m0) < 0) + data.swap(m1, m0); + // now data[m0] <= data[m1] <= data[m2] + } + + private static void swapRange(Sortable data, int a, int b, int n) + { + for (int i = 0; i < n; i++) + data.swap(a+i, b+i); + } + + private static void doPivot(Sortable data, int lo, int hi, int[] result) + { + int m = lo + (hi-lo)/2; // Written like this to avoid integer overflow. + if (hi-lo > 40) { + // Tukey's ``Ninther,'' median of three medians of three. + int s = (hi - lo) / 8; + medianOfThree(data, lo, lo+s, lo+2*s); + medianOfThree(data, m, m-s, m+s); + medianOfThree(data, hi-1, hi-1-s, hi-1-2*s); + } + medianOfThree(data, lo, m, hi-1); + + // Invariants are: + // data[lo] = pivot (set up by ChoosePivot) + // data[lo <= i < a] = pivot + // data[a <= i < b] < pivot + // data[b <= i < c] is unexamined + // data[c <= i < d] > pivot + // data[d <= i < hi] = pivot + // + // Once b meets c, can swap the "= pivot" sections + // into the middle of the slice. + int pivot = lo; + int a = lo+1, b = lo+1, c = hi, d =hi; + while (true) + { + while (b < c) + { + int cmp = data.compare(b, pivot); + if (cmp < 0) // data[b] < pivot + { + b++; + } + else if (cmp == 0) // data[b] = pivot + { + data.swap(a, b); + a++; + b++; + } + else + { + break; + } + } + + while (b < c) + { + int cmp = data.compare(pivot, c-1); + if (cmp < 0) // data[c-1] > pivot + { + c--; + } + else if (cmp == 0) // data[c-1] = pivot + { + data.swap(c-1, d-1); + c--; + d--; + } + else + { + break; + } + } + + if (b >= c) + break; + + // data[b] > pivot; data[c-1] < pivot + data.swap(b, c-1); + b++; + c--; + } + + int n = Math.min(b-a, a-lo); + swapRange(data, lo, b-n, n); + + n = Math.min(hi-d, d-c); + swapRange(data, c, hi-n, n); + + result[0] = lo + b - a; + result[1] = hi - (d - c); + } + + private static void quickSort(Sortable data, int a, int b, int maxDepth) + { + int[] buffer = new int[2]; + + while (b-a > 7) + { + if (maxDepth == 0) + { + heapSort(data, a, b); + return; + } + + maxDepth--; + + doPivot(data, a, b, buffer); + int mlo = buffer[0]; + int mhi = buffer[1]; + // Avoiding recursion on the larger subproblem guarantees + // a stack depth of at most lg(b-a). + if (mlo-a < b-mhi) + { + quickSort(data, a, mlo, maxDepth); + a = mhi; // i.e., quickSort(data, mhi, b) + } + else + { + quickSort(data, mhi, b, maxDepth); + b = mlo; // i.e., quickSort(data, a, mlo) + } + } + + if (b-a > 1) + insertionSort(data, a, b); + } +} diff --git a/src/java/org/apache/cassandra/utils/btree/BTree.java b/src/java/org/apache/cassandra/utils/btree/BTree.java index 1145d127f5..bf68ffa000 100644 --- a/src/java/org/apache/cassandra/utils/btree/BTree.java +++ b/src/java/org/apache/cassandra/utils/btree/BTree.java @@ -26,8 +26,6 @@ import java.util.Queue; import org.apache.cassandra.utils.ObjectSizes; -import static org.apache.cassandra.utils.btree.UpdateFunction.NoOp; - public class BTree { /** @@ -79,21 +77,18 @@ public class BTree return EMPTY_LEAF; } - public static Object[] build(Collection source, Comparator comparator, boolean sorted, UpdateFunction updateF) + public static Object[] build(Collection source, UpdateFunction updateF) { - return build(source, source.size(), comparator, sorted, updateF); + return build(source, source.size(), updateF); } /** * Creates a BTree containing all of the objects in the provided collection * * @param source the items to build the tree with - * @param comparator the comparator that defines the ordering over the items in the tree - * @param sorted if false, the collection will be copied and sorted to facilitate construction - * @param * @return */ - public static Object[] build(Iterable source, int size, Comparator comparator, boolean sorted, UpdateFunction updateF) + public static Object[] build(Iterable source, int size, UpdateFunction updateF) { if (size < FAN_FACTOR) { @@ -101,27 +96,13 @@ public class BTree V[] values = (V[]) new Object[size + (size & 1)]; { int i = 0; - for (V v : source) - values[i++] = v; - } - - // inline sorting since we're already calling toArray - if (!sorted) - Arrays.sort(values, 0, size, comparator); - - // if updateF is specified - if (updateF != null) - { - for (int i = 0 ; i < size ; i++) - values[i] = updateF.apply(values[i]); - updateF.allocated(ObjectSizes.sizeOfArray(values)); + for (K k : source) + values[i++] = updateF.apply(k); } + updateF.allocated(ObjectSizes.sizeOfArray(values)); return values; } - if (!sorted) - source = sorted(source, comparator, size); - Queue queue = modifier.get(); Builder builder = queue.poll(); if (builder == null) @@ -131,28 +112,12 @@ public class BTree return btree; } - /** - * Returns a new BTree with the provided set inserting/replacing as necessary any equal items - * - * @param btree the tree to update - * @param comparator the comparator that defines the ordering over the items in the tree - * @param updateWith the items to either insert / update - * @param updateWithIsSorted if false, updateWith will be copied and sorted to facilitate construction - * @param - * @return - */ - public static Object[] update(Object[] btree, Comparator comparator, Collection updateWith, boolean updateWithIsSorted) + public static Object[] update(Object[] btree, + Comparator comparator, + Collection updateWith, + UpdateFunction updateF) { - return update(btree, comparator, updateWith, updateWithIsSorted, NoOp.instance()); - } - - public static Object[] update(Object[] btree, - Comparator comparator, - Collection updateWith, - boolean updateWithIsSorted, - UpdateFunction updateF) - { - return update(btree, comparator, updateWith, updateWith.size(), updateWithIsSorted, updateF); + return update(btree, comparator, updateWith, updateWith.size(), updateF); } /** @@ -161,23 +126,19 @@ public class BTree * @param btree the tree to update * @param comparator the comparator that defines the ordering over the items in the tree * @param updateWith the items to either insert / update - * @param updateWithIsSorted if false, updateWith will be copied and sorted to facilitate construction + * @param updateWithLength then number of elements in updateWith * @param updateF the update function to apply to any pairs we are swapping, and maybe abort early * @param * @return */ - public static Object[] update(Object[] btree, - Comparator comparator, - Iterable updateWith, - int updateWithLength, - boolean updateWithIsSorted, - UpdateFunction updateF) + public static Object[] update(Object[] btree, + Comparator comparator, + Iterable updateWith, + int updateWithLength, + UpdateFunction updateF) { if (btree.length == 0) - return build(updateWith, updateWithLength, comparator, updateWithIsSorted, updateF); - - if (!updateWithIsSorted) - updateWith = sorted(updateWith, comparator, updateWithLength); + return build(updateWith, updateWithLength, updateF); Queue queue = modifier.get(); Builder builder = queue.poll(); @@ -361,17 +322,6 @@ public class BTree } }; - // return a sorted collection - private static Collection sorted(Iterable source, Comparator comparator, int size) - { - V[] vs = (V[]) new Object[size]; - int i = 0; - for (V v : source) - vs[i++] = v; - Arrays.sort(vs, comparator); - return Arrays.asList(vs); - } - /** simple static wrapper to calls to cmp.compare() which checks if either a or b are Special (i.e. represent an infinity) */ // TODO : cheaper to check for POSITIVE/NEGATIVE infinity in callers, rather than here static int compare(Comparator cmp, Object a, Object b) diff --git a/src/java/org/apache/cassandra/utils/btree/BTreeSearchIterator.java b/src/java/org/apache/cassandra/utils/btree/BTreeSearchIterator.java index 7a83238c57..403f234d4b 100644 --- a/src/java/org/apache/cassandra/utils/btree/BTreeSearchIterator.java +++ b/src/java/org/apache/cassandra/utils/btree/BTreeSearchIterator.java @@ -25,43 +25,76 @@ import static org.apache.cassandra.utils.btree.BTree.getKeyEnd; public class BTreeSearchIterator extends Path implements SearchIterator { - final Comparator comparator; - public BTreeSearchIterator(Object[] btree, Comparator comparator) + final boolean forwards; + + public BTreeSearchIterator(Object[] btree, Comparator comparator, boolean forwards) { init(btree); + if (!forwards) + this.indexes[0] = (byte)(getKeyEnd(path[0]) - 1); this.comparator = comparator; + this.forwards = forwards; } public V next(K target) { - while (depth > 0) + // We could probably avoid some of the repetition but leaving that for later. + if (forwards) { - byte successorParentDepth = findSuccessorParentDepth(); - if (successorParentDepth < 0) - break; // we're in last section of tree, so can only search down - int successorParentIndex = indexes[successorParentDepth] + 1; - Object[] successParentNode = path[successorParentDepth]; - Object successorParentKey = successParentNode[successorParentIndex]; - int c = BTree.compare(comparator, target, successorParentKey); - if (c < 0) - break; - if (c == 0) + while (depth > 0) { + byte successorParentDepth = findSuccessorParentDepth(); + if (successorParentDepth < 0) + break; // we're in last section of tree, so can only search down + int successorParentIndex = indexes[successorParentDepth] + 1; + Object[] successParentNode = path[successorParentDepth]; + Object successorParentKey = successParentNode[successorParentIndex]; + int c = BTree.compare(comparator, target, successorParentKey); + if (c < 0) + break; + if (c == 0) + { + depth = successorParentDepth; + indexes[successorParentDepth]++; + return (V) successorParentKey; + } depth = successorParentDepth; indexes[successorParentDepth]++; - return (V) successorParentKey; } - depth = successorParentDepth; - indexes[successorParentDepth]++; + if (find(comparator, target, Op.CEIL, true)) + return (V) currentKey(); + } + else + { + while (depth > 0) + { + byte predecessorParentDepth = findPredecessorParentDepth(); + if (predecessorParentDepth < 0) + break; // we're in last section of tree, so can only search down + int predecessorParentIndex = indexes[predecessorParentDepth] - 1; + Object[] predecessParentNode = path[predecessorParentDepth]; + Object predecessorParentKey = predecessParentNode[predecessorParentIndex]; + int c = BTree.compare(comparator, target, predecessorParentKey); + if (c > 0) + break; + if (c == 0) + { + depth = predecessorParentDepth; + indexes[predecessorParentDepth]--; + return (V) predecessorParentKey; + } + depth = predecessorParentDepth; + indexes[predecessorParentDepth]--; + } + if (find(comparator, target, Op.FLOOR, false)) + return (V) currentKey(); } - if (find(comparator, target, Op.CEIL, true)) - return (V) currentKey(); return null; } public boolean hasNext() { - return depth != 0 || indexes[0] != getKeyEnd(path[0]); + return depth != 0 || indexes[0] != (forwards ? getKeyEnd(path[0]) : -1); } } diff --git a/src/java/org/apache/cassandra/utils/btree/BTreeSet.java b/src/java/org/apache/cassandra/utils/btree/BTreeSet.java deleted file mode 100644 index d80b32eaa0..0000000000 --- a/src/java/org/apache/cassandra/utils/btree/BTreeSet.java +++ /dev/null @@ -1,383 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - */ -package org.apache.cassandra.utils.btree; - -import java.util.Arrays; -import java.util.Collection; -import java.util.Comparator; -import java.util.Iterator; -import java.util.NavigableSet; -import java.util.SortedSet; - -public class BTreeSet implements NavigableSet -{ - protected final Comparator comparator; - protected final Object[] tree; - - public BTreeSet(Object[] tree, Comparator comparator) - { - this.tree = tree; - this.comparator = comparator; - } - - public BTreeSet update(Collection updateWith, boolean isSorted) - { - return new BTreeSet<>(BTree.update(tree, comparator, updateWith, isSorted, UpdateFunction.NoOp.instance()), comparator); - } - - @Override - public Comparator comparator() - { - return comparator; - } - - protected Cursor slice(boolean forwards, boolean permitInversion) - { - return BTree.slice(tree, forwards); - } - - @Override - public int size() - { - return slice(true, false).count(); - } - - @Override - public boolean isEmpty() - { - return slice(true, false).hasNext(); - } - - @Override - public Iterator iterator() - { - return slice(true, true); - } - - @Override - public Iterator descendingIterator() - { - return slice(false, true); - } - - @Override - public Object[] toArray() - { - return toArray(new Object[0]); - } - - @Override - public T[] toArray(T[] a) - { - int size = size(); - if (a.length < size) - a = Arrays.copyOf(a, size); - int i = 0; - for (V v : this) - a[i++] = (T) v; - return a; - } - - @Override - public NavigableSet subSet(V fromElement, boolean fromInclusive, V toElement, boolean toInclusive) - { - return new BTreeRange<>(tree, comparator, fromElement, fromInclusive, toElement, toInclusive); - } - - @Override - public NavigableSet headSet(V toElement, boolean inclusive) - { - return new BTreeRange<>(tree, comparator, null, true, toElement, inclusive); - } - - @Override - public NavigableSet tailSet(V fromElement, boolean inclusive) - { - return new BTreeRange<>(tree, comparator, fromElement, inclusive, null, true); - } - - @Override - public SortedSet subSet(V fromElement, V toElement) - { - return subSet(fromElement, true, toElement, false); - } - - @Override - public SortedSet headSet(V toElement) - { - return headSet(toElement, false); - } - - @Override - public SortedSet tailSet(V fromElement) - { - return tailSet(fromElement, true); - } - - @Override - public V first() - { - throw new UnsupportedOperationException(); - } - - @Override - public V last() - { - throw new UnsupportedOperationException(); - } - - @Override - public boolean addAll(Collection c) - { - throw new UnsupportedOperationException(); - } - - @Override - public boolean retainAll(Collection c) - { - throw new UnsupportedOperationException(); - } - - @Override - public boolean removeAll(Collection c) - { - throw new UnsupportedOperationException(); - } - - @Override - public void clear() - { - throw new UnsupportedOperationException(); - } - - @Override - public V pollFirst() - { - throw new UnsupportedOperationException(); - } - - @Override - public V pollLast() - { - throw new UnsupportedOperationException(); - } - - @Override - public boolean add(V v) - { - throw new UnsupportedOperationException(); - } - - @Override - public boolean remove(Object o) - { - throw new UnsupportedOperationException(); - } - - @Override - public V lower(V v) - { - throw new UnsupportedOperationException(); - } - - @Override - public V floor(V v) - { - throw new UnsupportedOperationException(); - } - - @Override - public V ceiling(V v) - { - throw new UnsupportedOperationException(); - } - - @Override - public V higher(V v) - { - throw new UnsupportedOperationException(); - } - - @Override - public boolean contains(Object o) - { - throw new UnsupportedOperationException(); - } - - @Override - public boolean containsAll(Collection c) - { - throw new UnsupportedOperationException(); - } - - @Override - public NavigableSet descendingSet() - { - return new BTreeRange<>(this.tree, this.comparator).descendingSet(); - } - - public static class BTreeRange extends BTreeSet implements NavigableSet - { - - protected final V lowerBound, upperBound; - protected final boolean inclusiveLowerBound, inclusiveUpperBound; - - BTreeRange(Object[] tree, Comparator comparator) - { - this(tree, comparator, null, true, null, true); - } - - BTreeRange(BTreeRange from) - { - this(from.tree, from.comparator, from.lowerBound, from.inclusiveLowerBound, from.upperBound, from.inclusiveUpperBound); - } - - BTreeRange(Object[] tree, Comparator comparator, V lowerBound, boolean inclusiveLowerBound, V upperBound, boolean inclusiveUpperBound) - { - super(tree, comparator); - this.lowerBound = lowerBound; - this.upperBound = upperBound; - this.inclusiveLowerBound = inclusiveLowerBound; - this.inclusiveUpperBound = inclusiveUpperBound; - } - - // narrowing range constructor - makes this the intersection of the two ranges over the same tree b - BTreeRange(BTreeRange a, BTreeRange b) - { - super(a.tree, a.comparator); - assert a.tree == b.tree; - final BTreeRange lb, ub; - - if (a.lowerBound == null) - { - lb = b; - } - else if (b.lowerBound == null) - { - lb = a; - } - else - { - int c = comparator.compare(a.lowerBound, b.lowerBound); - if (c < 0) - lb = b; - else if (c > 0) - lb = a; - else if (!a.inclusiveLowerBound) - lb = a; - else - lb = b; - } - - if (a.upperBound == null) - { - ub = b; - } - else if (b.upperBound == null) - { - ub = a; - } - else - { - int c = comparator.compare(b.upperBound, a.upperBound); - if (c < 0) - ub = b; - else if (c > 0) - ub = a; - else if (!a.inclusiveUpperBound) - ub = a; - else - ub = b; - } - - lowerBound = lb.lowerBound; - inclusiveLowerBound = lb.inclusiveLowerBound; - upperBound = ub.upperBound; - inclusiveUpperBound = ub.inclusiveUpperBound; - } - - @Override - protected Cursor slice(boolean forwards, boolean permitInversion) - { - return BTree.slice(tree, comparator, lowerBound, inclusiveLowerBound, upperBound, inclusiveUpperBound, forwards); - } - - @Override - public NavigableSet subSet(V fromElement, boolean fromInclusive, V toElement, boolean toInclusive) - { - return new BTreeRange<>(this, new BTreeRange<>(tree, comparator, fromElement, fromInclusive, toElement, toInclusive)); - } - - @Override - public NavigableSet headSet(V toElement, boolean inclusive) - { - return new BTreeRange<>(this, new BTreeRange<>(tree, comparator, lowerBound, true, toElement, inclusive)); - } - - @Override - public NavigableSet tailSet(V fromElement, boolean inclusive) - { - return new BTreeRange<>(this, new BTreeRange<>(tree, comparator, fromElement, inclusive, null, true)); - } - - @Override - public NavigableSet descendingSet() - { - return new BTreeDescRange<>(this); - } - } - - public static class BTreeDescRange extends BTreeRange - { - BTreeDescRange(BTreeRange from) - { - super(from.tree, from.comparator, from.lowerBound, from.inclusiveLowerBound, from.upperBound, from.inclusiveUpperBound); - } - - @Override - protected Cursor slice(boolean forwards, boolean permitInversion) - { - return super.slice(permitInversion ? !forwards : forwards, false); - } - - @Override - public NavigableSet subSet(V fromElement, boolean fromInclusive, V toElement, boolean toInclusive) - { - return super.subSet(toElement, toInclusive, fromElement, fromInclusive).descendingSet(); - } - - @Override - public NavigableSet headSet(V toElement, boolean inclusive) - { - return super.tailSet(toElement, inclusive).descendingSet(); - } - - @Override - public NavigableSet tailSet(V fromElement, boolean inclusive) - { - return super.headSet(fromElement, inclusive).descendingSet(); - } - - @Override - public NavigableSet descendingSet() - { - return new BTreeRange<>(this); - } - } -} diff --git a/src/java/org/apache/cassandra/utils/btree/Builder.java b/src/java/org/apache/cassandra/utils/btree/Builder.java index ab7801688b..b835554eb3 100644 --- a/src/java/org/apache/cassandra/utils/btree/Builder.java +++ b/src/java/org/apache/cassandra/utils/btree/Builder.java @@ -54,14 +54,14 @@ final class Builder * we assume @param source has been sorted, e.g. by BTree.update, so the update of each key resumes where * the previous left off. */ - public Object[] update(Object[] btree, Comparator comparator, Iterable source, UpdateFunction updateF) + public Object[] update(Object[] btree, Comparator comparator, Iterable source, UpdateFunction updateF) { assert updateF != null; NodeBuilder current = rootBuilder; current.reset(btree, POSITIVE_INFINITY, updateF, comparator); - for (V key : source) + for (K key : source) { while (true) { @@ -96,7 +96,7 @@ final class Builder return r; } - public Object[] build(Iterable source, UpdateFunction updateF, int size) + public Object[] build(Iterable source, UpdateFunction updateF, int size) { assert updateF != null; @@ -107,7 +107,7 @@ final class Builder current = current.ensureChild(); current.reset(EMPTY_LEAF, POSITIVE_INFINITY, updateF, null); - for (V key : source) + for (K key : source) current.addNewKey(updateF.apply(key)); current = current.ascendToRoot(); diff --git a/src/java/org/apache/cassandra/utils/btree/Path.java b/src/java/org/apache/cassandra/utils/btree/Path.java index b1b0e03c86..9b6789c314 100644 --- a/src/java/org/apache/cassandra/utils/btree/Path.java +++ b/src/java/org/apache/cassandra/utils/btree/Path.java @@ -104,8 +104,7 @@ public class Path // search Object[] node = path[depth]; - int lb = indexes[depth]; - assert lb == 0 || forwards; + int lb = forwards ? indexes[depth] : 0; pop(); if (target instanceof BTree.Special) @@ -223,6 +222,20 @@ public class Path return -1; } + byte findPredecessorParentDepth() + { + byte depth = this.depth; + depth--; + while (depth >= 0) + { + int ub = indexes[depth] - 1; + if (ub >= 0) + return depth; + depth--; + } + return -1; + } + // move to the next key in the tree void successor() { diff --git a/src/java/org/apache/cassandra/utils/btree/UpdateFunction.java b/src/java/org/apache/cassandra/utils/btree/UpdateFunction.java index 9f450317a5..93c02aea91 100644 --- a/src/java/org/apache/cassandra/utils/btree/UpdateFunction.java +++ b/src/java/org/apache/cassandra/utils/btree/UpdateFunction.java @@ -22,17 +22,15 @@ import com.google.common.base.Function; /** * An interface defining a function to be applied to both the object we are replacing in a BTree and * the object that is intended to replace it, returning the object to actually replace it. - * - * @param */ -public interface UpdateFunction extends Function +public interface UpdateFunction extends Function { /** * @param replacing the value in the original tree we have matched * @param update the value in the updating collection that matched * @return the value to insert into the new tree */ - V apply(V replacing, V update); + V apply(V replacing, K update); /** * @return true if we should fail the update @@ -44,37 +42,4 @@ public interface UpdateFunction extends Function */ void allocated(long heapSize); - public static final class NoOp implements UpdateFunction - { - - private static final NoOp INSTANCE = new NoOp(); - public static NoOp instance() - { - return INSTANCE; - } - - private NoOp() - { - } - - public V apply(V replacing, V update) - { - return update; - } - - public V apply(V update) - { - return update; - } - - public boolean abortEarly() - { - return false; - } - - public void allocated(long heapSize) - { - } - } - } diff --git a/src/java/org/apache/cassandra/utils/concurrent/Accumulator.java b/src/java/org/apache/cassandra/utils/concurrent/Accumulator.java index baecb34739..e80faca806 100644 --- a/src/java/org/apache/cassandra/utils/concurrent/Accumulator.java +++ b/src/java/org/apache/cassandra/utils/concurrent/Accumulator.java @@ -100,6 +100,11 @@ public class Accumulator implements Iterable return presentCount; } + public int capacity() + { + return values.length; + } + public Iterator iterator() { return new Iterator() diff --git a/src/java/org/apache/cassandra/utils/memory/HeapPool.java b/src/java/org/apache/cassandra/utils/memory/HeapPool.java index 3d3ca09706..19f81be41a 100644 --- a/src/java/org/apache/cassandra/utils/memory/HeapPool.java +++ b/src/java/org/apache/cassandra/utils/memory/HeapPool.java @@ -22,7 +22,6 @@ import java.nio.ByteBuffer; import java.util.ArrayList; import java.util.List; -import org.apache.cassandra.db.Cell; import org.apache.cassandra.db.DecoratedKey; import org.apache.cassandra.utils.concurrent.OpOrder; @@ -40,63 +39,66 @@ public class HeapPool extends MemtablePool public MemtableAllocator newAllocator() { - return new Allocator(this); + // TODO + throw new UnsupportedOperationException(); + //return new Allocator(this); } - public static class Allocator extends MemtableBufferAllocator - { - Allocator(HeapPool pool) - { - super(pool.onHeap.newAllocator(), pool.offHeap.newAllocator()); - } + // TODO + //public static class Allocator extends MemtableBufferAllocator + //{ + // Allocator(HeapPool pool) + // { + // super(pool.onHeap.newAllocator(), pool.offHeap.newAllocator()); + // } - public ByteBuffer allocate(int size, OpOrder.Group opGroup) - { - super.onHeap().allocate(size, opGroup); - return ByteBuffer.allocate(size); - } + // public ByteBuffer allocate(int size, OpOrder.Group opGroup) + // { + // super.onHeap().allocate(size, opGroup); + // return ByteBuffer.allocate(size); + // } - public DataReclaimer reclaimer() - { - return new Reclaimer(); - } + // public DataReclaimer reclaimer() + // { + // return new Reclaimer(); + // } - private class Reclaimer implements DataReclaimer - { - List delayed; + // private class Reclaimer implements DataReclaimer + // { + // List delayed; - public Reclaimer reclaim(Cell cell) - { - if (delayed == null) - delayed = new ArrayList<>(); - delayed.add(cell); - return this; - } + // public Reclaimer reclaim(Cell cell) + // { + // if (delayed == null) + // delayed = new ArrayList<>(); + // delayed.add(cell); + // return this; + // } - public Reclaimer reclaimImmediately(Cell cell) - { - onHeap().release(cell.name().dataSize() + cell.value().remaining()); - return this; - } + // public Reclaimer reclaimImmediately(Cell cell) + // { + // onHeap().release(cell.name().dataSize() + cell.value().remaining()); + // return this; + // } - public Reclaimer reclaimImmediately(DecoratedKey key) - { - onHeap().release(key.getKey().remaining()); - return this; - } + // public Reclaimer reclaimImmediately(DecoratedKey key) + // { + // onHeap().release(key.getKey().remaining()); + // return this; + // } - public void cancel() - { - if (delayed != null) - delayed.clear(); - } + // public void cancel() + // { + // if (delayed != null) + // delayed.clear(); + // } - public void commit() - { - if (delayed != null) - for (Cell cell : delayed) - reclaimImmediately(cell); - } - } - } + // public void commit() + // { + // if (delayed != null) + // for (Cell cell : delayed) + // reclaimImmediately(cell); + // } + // } + //} } diff --git a/src/java/org/apache/cassandra/utils/memory/MemtableAllocator.java b/src/java/org/apache/cassandra/utils/memory/MemtableAllocator.java index e814b4daf9..1e0c11ee80 100644 --- a/src/java/org/apache/cassandra/utils/memory/MemtableAllocator.java +++ b/src/java/org/apache/cassandra/utils/memory/MemtableAllocator.java @@ -22,6 +22,7 @@ import java.util.concurrent.atomic.AtomicLongFieldUpdater; import org.apache.cassandra.config.CFMetaData; import org.apache.cassandra.db.*; +import org.apache.cassandra.db.rows.*; import org.apache.cassandra.utils.concurrent.OpOrder; import org.apache.cassandra.utils.concurrent.WaitQueue; @@ -58,10 +59,8 @@ public abstract class MemtableAllocator this.offHeap = offHeap; } - public abstract Cell clone(Cell cell, CFMetaData cfm, OpOrder.Group writeOp); - public abstract CounterCell clone(CounterCell cell, CFMetaData cfm, OpOrder.Group writeOp); - public abstract DeletedCell clone(DeletedCell cell, CFMetaData cfm, OpOrder.Group writeOp); - public abstract ExpiringCell clone(ExpiringCell cell, CFMetaData cfm, OpOrder.Group writeOp); + public abstract MemtableRowData.ReusableRow newReusableRow(); + public abstract RowAllocator newRowAllocator(CFMetaData cfm, OpOrder.Group writeOp); public abstract DecoratedKey clone(DecoratedKey key, OpOrder.Group opGroup); public abstract DataReclaimer reclaimer(); @@ -104,10 +103,16 @@ public abstract class MemtableAllocator return state == LifeCycle.LIVE; } + public static interface RowAllocator extends Row.Writer + { + public void allocateNewRow(int clusteringSize, Columns columns, boolean isStatic); + public MemtableRowData allocatedRowData(); + } + public static interface DataReclaimer { - public DataReclaimer reclaim(Cell cell); - public DataReclaimer reclaimImmediately(Cell cell); + public DataReclaimer reclaim(MemtableRowData row); + public DataReclaimer reclaimImmediately(MemtableRowData row); public DataReclaimer reclaimImmediately(DecoratedKey key); public void cancel(); public void commit(); @@ -115,12 +120,12 @@ public abstract class MemtableAllocator public static final DataReclaimer NO_OP = new DataReclaimer() { - public DataReclaimer reclaim(Cell cell) + public DataReclaimer reclaim(MemtableRowData update) { return this; } - public DataReclaimer reclaimImmediately(Cell cell) + public DataReclaimer reclaimImmediately(MemtableRowData update) { return this; } diff --git a/src/java/org/apache/cassandra/utils/memory/MemtableBufferAllocator.java b/src/java/org/apache/cassandra/utils/memory/MemtableBufferAllocator.java index 7034d76bb5..144f439898 100644 --- a/src/java/org/apache/cassandra/utils/memory/MemtableBufferAllocator.java +++ b/src/java/org/apache/cassandra/utils/memory/MemtableBufferAllocator.java @@ -20,12 +20,9 @@ package org.apache.cassandra.utils.memory; import java.nio.ByteBuffer; import org.apache.cassandra.config.CFMetaData; -import org.apache.cassandra.db.BufferDecoratedKey; -import org.apache.cassandra.db.Cell; -import org.apache.cassandra.db.CounterCell; -import org.apache.cassandra.db.DecoratedKey; -import org.apache.cassandra.db.DeletedCell; -import org.apache.cassandra.db.ExpiringCell; +import org.apache.cassandra.config.ColumnDefinition; +import org.apache.cassandra.db.*; +import org.apache.cassandra.db.rows.*; import org.apache.cassandra.utils.concurrent.OpOrder; public abstract class MemtableBufferAllocator extends MemtableAllocator @@ -36,24 +33,14 @@ public abstract class MemtableBufferAllocator extends MemtableAllocator super(onHeap, offHeap); } - public Cell clone(Cell cell, CFMetaData cfm, OpOrder.Group writeOp) + public MemtableRowData.ReusableRow newReusableRow() { - return cell.localCopy(cfm, allocator(writeOp)); + return MemtableRowData.BufferRowData.createReusableRow(); } - public CounterCell clone(CounterCell cell, CFMetaData cfm, OpOrder.Group writeOp) + public RowAllocator newRowAllocator(CFMetaData cfm, OpOrder.Group writeOp) { - return cell.localCopy(cfm, allocator(writeOp)); - } - - public DeletedCell clone(DeletedCell cell, CFMetaData cfm, OpOrder.Group writeOp) - { - return cell.localCopy(cfm, allocator(writeOp)); - } - - public ExpiringCell clone(ExpiringCell cell, CFMetaData cfm, OpOrder.Group writeOp) - { - return cell.localCopy(cfm, allocator(writeOp)); + return new RowBufferAllocator(allocator(writeOp), cfm.isCounter()); } public DecoratedKey clone(DecoratedKey key, OpOrder.Group writeOp) @@ -67,4 +54,71 @@ public abstract class MemtableBufferAllocator extends MemtableAllocator { return new ContextAllocator(writeOp, this); } + + private static class RowBufferAllocator extends RowDataBlock.Writer implements RowAllocator + { + private final AbstractAllocator allocator; + private final boolean isCounter; + + private MemtableRowData.BufferClustering clustering; + private int clusteringIdx; + private LivenessInfo info; + private DeletionTime deletion; + private RowDataBlock data; + + private RowBufferAllocator(AbstractAllocator allocator, boolean isCounter) + { + super(true); + this.allocator = allocator; + this.isCounter = isCounter; + } + + public void allocateNewRow(int clusteringSize, Columns columns, boolean isStatic) + { + data = new RowDataBlock(columns, 1, false, isCounter); + clustering = isStatic ? null : new MemtableRowData.BufferClustering(clusteringSize); + clusteringIdx = 0; + updateWriter(data); + } + + public MemtableRowData allocatedRowData() + { + MemtableRowData row = new MemtableRowData.BufferRowData(clustering == null ? Clustering.STATIC_CLUSTERING : clustering, + info, + deletion, + data); + + clustering = null; + info = LivenessInfo.NONE; + deletion = DeletionTime.LIVE; + data = null; + + return row; + } + + public void writeClusteringValue(ByteBuffer value) + { + clustering.setClusteringValue(clusteringIdx++, value == null ? null : allocator.clone(value)); + } + + public void writePartitionKeyLivenessInfo(LivenessInfo info) + { + this.info = info; + } + + public void writeRowDeletion(DeletionTime deletion) + { + this.deletion = deletion; + } + + @Override + public void writeCell(ColumnDefinition column, boolean isCounter, ByteBuffer value, LivenessInfo info, CellPath path) + { + ByteBuffer v = allocator.clone(value); + if (column.isComplex()) + complexWriter.addCell(column, v, info, MemtableRowData.BufferCellPath.clone(path, allocator)); + else + simpleWriter.addCell(column, v, info); + } + } } diff --git a/src/java/org/apache/cassandra/utils/memory/NativeAllocator.java b/src/java/org/apache/cassandra/utils/memory/NativeAllocator.java index 88846c5f40..7ca859db40 100644 --- a/src/java/org/apache/cassandra/utils/memory/NativeAllocator.java +++ b/src/java/org/apache/cassandra/utils/memory/NativeAllocator.java @@ -25,16 +25,9 @@ import java.util.concurrent.atomic.AtomicInteger; import java.util.concurrent.atomic.AtomicReference; import org.apache.cassandra.config.CFMetaData; -import org.apache.cassandra.db.Cell; -import org.apache.cassandra.db.CounterCell; import org.apache.cassandra.db.DecoratedKey; -import org.apache.cassandra.db.DeletedCell; -import org.apache.cassandra.db.ExpiringCell; -import org.apache.cassandra.db.NativeCell; -import org.apache.cassandra.db.NativeCounterCell; import org.apache.cassandra.db.NativeDecoratedKey; -import org.apache.cassandra.db.NativeDeletedCell; -import org.apache.cassandra.db.NativeExpiringCell; +import org.apache.cassandra.db.rows.MemtableRowData; import org.apache.cassandra.utils.concurrent.OpOrder; public class NativeAllocator extends MemtableAllocator @@ -60,28 +53,16 @@ public class NativeAllocator extends MemtableAllocator super(pool.onHeap.newAllocator(), pool.offHeap.newAllocator()); } - @Override - public Cell clone(Cell cell, CFMetaData cfm, OpOrder.Group writeOp) + public MemtableRowData.ReusableRow newReusableRow() { - return new NativeCell(this, writeOp, cell); + // TODO + throw new UnsupportedOperationException(); } - @Override - public CounterCell clone(CounterCell cell, CFMetaData cfm, OpOrder.Group writeOp) + public RowAllocator newRowAllocator(CFMetaData cfm, OpOrder.Group writeOp) { - return new NativeCounterCell(this, writeOp, cell); - } - - @Override - public DeletedCell clone(DeletedCell cell, CFMetaData cfm, OpOrder.Group writeOp) - { - return new NativeDeletedCell(this, writeOp, cell); - } - - @Override - public ExpiringCell clone(ExpiringCell cell, CFMetaData cfm, OpOrder.Group writeOp) - { - return new NativeExpiringCell(this, writeOp, cell); + // TODO + throw new UnsupportedOperationException(); } public DecoratedKey clone(DecoratedKey key, OpOrder.Group writeOp) diff --git a/test/burn/org/apache/cassandra/utils/LongBTreeTest.java b/test/burn/org/apache/cassandra/utils/LongBTreeTest.java deleted file mode 100644 index 96419309ea..0000000000 --- a/test/burn/org/apache/cassandra/utils/LongBTreeTest.java +++ /dev/null @@ -1,502 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - */ -package org.apache.cassandra.utils; - -import java.util.ArrayList; -import java.util.Arrays; -import java.util.Comparator; -import java.util.Iterator; -import java.util.List; -import java.util.NavigableMap; -import java.util.NavigableSet; -import java.util.Random; -import java.util.TreeMap; -import java.util.TreeSet; -import java.util.concurrent.Callable; -import java.util.concurrent.CountDownLatch; -import java.util.concurrent.ExecutionException; -import java.util.concurrent.ExecutorService; -import java.util.concurrent.Executors; -import java.util.concurrent.ThreadLocalRandom; -import java.util.concurrent.TimeUnit; -import java.util.concurrent.atomic.AtomicLong; - -import com.google.common.base.Function; -import com.google.common.base.Predicate; -import com.google.common.collect.Iterables; -import com.google.common.util.concurrent.Futures; -import com.google.common.util.concurrent.ListenableFuture; -import com.google.common.util.concurrent.ListenableFutureTask; -import org.junit.Assert; -import org.junit.Test; - - -import com.codahale.metrics.MetricRegistry; -import com.codahale.metrics.Snapshot; -import com.codahale.metrics.Timer; -import org.apache.cassandra.concurrent.NamedThreadFactory; -import org.apache.cassandra.utils.btree.BTree; -import org.apache.cassandra.utils.btree.BTreeSearchIterator; -import org.apache.cassandra.utils.btree.BTreeSet; -import org.apache.cassandra.utils.btree.UpdateFunction; - -// TODO : should probably lower fan-factor for tests to make them more intensive -public class LongBTreeTest -{ - - private static final MetricRegistry metrics = new MetricRegistry(); - private static final Timer BTREE_TIMER = metrics.timer(MetricRegistry.name(BTree.class, "BTREE")); - private static final Timer TREE_TIMER = metrics.timer(MetricRegistry.name(BTree.class, "TREE")); - private static final ExecutorService MODIFY = Executors.newFixedThreadPool(Runtime.getRuntime().availableProcessors(), new NamedThreadFactory("MODIFY")); - private static final ExecutorService COMPARE = Executors.newFixedThreadPool(Runtime.getRuntime().availableProcessors(), new NamedThreadFactory("COMPARE")); - private static final RandomAbort SPORADIC_ABORT = new RandomAbort<>(new Random(), 0.0001f); - - static - { - System.setProperty("cassandra.btree.fanfactor", "4"); - } - - @Test - public void testOversizedMiddleInsert() - { - TreeSet canon = new TreeSet<>(); - for (int i = 0 ; i < 10000000 ; i++) - canon.add(i); - Object[] btree = BTree.build(Arrays.asList(Integer.MIN_VALUE, Integer.MAX_VALUE), ICMP, true, null); - btree = BTree.update(btree, ICMP, canon, true); - canon.add(Integer.MIN_VALUE); - canon.add(Integer.MAX_VALUE); - Assert.assertTrue(BTree.isWellFormed(btree, ICMP)); - testEqual("Oversize", BTree.slice(btree, true), canon.iterator()); - } - - @Test - public void testIndividualInsertsSmallOverlappingRange() throws ExecutionException, InterruptedException - { - testInsertions(10000000, 50, 1, 1, true); - } - - @Test - public void testBatchesSmallOverlappingRange() throws ExecutionException, InterruptedException - { - testInsertions(10000000, 50, 1, 5, true); - } - - @Test - public void testIndividualInsertsMediumSparseRange() throws ExecutionException, InterruptedException - { - testInsertions(10000000, 500, 10, 1, true); - } - - @Test - public void testBatchesMediumSparseRange() throws ExecutionException, InterruptedException - { - testInsertions(10000000, 500, 10, 10, true); - } - - @Test - public void testLargeBatchesLargeRange() throws ExecutionException, InterruptedException - { - testInsertions(100000000, 5000, 3, 100, true); - } - - @Test - public void testSlicingSmallRandomTrees() throws ExecutionException, InterruptedException - { - testInsertions(10000, 50, 10, 10, false); - } - - @Test - public void testSearchIterator() throws InterruptedException - { - int threads = Runtime.getRuntime().availableProcessors(); - final CountDownLatch latch = new CountDownLatch(threads); - final AtomicLong errors = new AtomicLong(); - final AtomicLong count = new AtomicLong(); - final int perThreadTrees = 100; - final int perTreeSelections = 100; - final long totalCount = threads * perThreadTrees * perTreeSelections; - for (int t = 0 ; t < threads ; t++) - { - MODIFY.execute(new Runnable() - { - public void run() - { - ThreadLocalRandom random = ThreadLocalRandom.current(); - for (int i = 0 ; i < perThreadTrees ; i++) - { - Object[] tree = randomTree(10000, random); - for (int j = 0 ; j < perTreeSelections ; j++) - { - BTreeSearchIterator searchIterator = new BTreeSearchIterator<>(tree, ICMP); - for (Integer key : randomSelection(tree, random)) - if (key != searchIterator.next(key)) - errors.incrementAndGet(); - searchIterator = new BTreeSearchIterator(tree, ICMP); - for (Integer key : randomMix(tree, random)) - if (key != searchIterator.next(key)) - if (BTree.find(tree, ICMP, key) == key) - errors.incrementAndGet(); - count.incrementAndGet(); - } - } - latch.countDown(); - } - }); - } - while (latch.getCount() > 0) - { - latch.await(10L, TimeUnit.SECONDS); - System.out.println(String.format("%.0f%% complete %s", 100 * count.get() / (double) totalCount, errors.get() > 0 ? ("Errors: " + errors.get()) : "")); - assert errors.get() == 0; - } - } - - private static void testInsertions(int totalCount, int perTestCount, int testKeyRatio, int modificationBatchSize, boolean quickEquality) throws ExecutionException, InterruptedException - { - int batchesPerTest = perTestCount / modificationBatchSize; - int maximumRunLength = 100; - int testKeyRange = perTestCount * testKeyRatio; - int tests = totalCount / perTestCount; - System.out.println(String.format("Performing %d tests of %d operations, with %.2f max size/key-range ratio in batches of ~%d ops", - tests, perTestCount, 1 / (float) testKeyRatio, modificationBatchSize)); - - // if we're not doing quick-equality, we can spam with garbage for all the checks we perform, so we'll split the work into smaller chunks - int chunkSize = quickEquality ? tests : (int) (100000 / Math.pow(perTestCount, 2)); - for (int chunk = 0 ; chunk < tests ; chunk += chunkSize) - { - final List>>> outer = new ArrayList<>(); - for (int i = 0 ; i < chunkSize ; i++) - { - outer.add(doOneTestInsertions(testKeyRange, maximumRunLength, modificationBatchSize, batchesPerTest, quickEquality)); - } - - final List> inner = new ArrayList<>(); - int complete = 0; - int reportInterval = totalCount / 100; - int lastReportAt = 0; - for (ListenableFutureTask>> f : outer) - { - inner.addAll(f.get()); - complete += perTestCount; - if (complete - lastReportAt >= reportInterval) - { - System.out.println(String.format("Completed %d of %d operations", (chunk * perTestCount) + complete, totalCount)); - lastReportAt = complete; - } - } - Futures.allAsList(inner).get(); - } - Snapshot snap = BTREE_TIMER.getSnapshot(); - System.out.println(String.format("btree : %.2fns, %.2fns, %.2fns", snap.getMedian(), snap.get95thPercentile(), snap.get999thPercentile())); - snap = TREE_TIMER.getSnapshot(); - System.out.println(String.format("snaptree: %.2fns, %.2fns, %.2fns", snap.getMedian(), snap.get95thPercentile(), snap.get999thPercentile())); - System.out.println("Done"); - } - - private static ListenableFutureTask>> doOneTestInsertions(final int upperBound, final int maxRunLength, final int averageModsPerIteration, final int iterations, final boolean quickEquality) - { - ListenableFutureTask>> f = ListenableFutureTask.create(new Callable>>() - { - @Override - public List> call() - { - final List> r = new ArrayList<>(); - NavigableMap canon = new TreeMap<>(); - Object[] btree = BTree.empty(); - final TreeMap buffer = new TreeMap<>(); - final Random rnd = new Random(); - for (int i = 0 ; i < iterations ; i++) - { - buffer.clear(); - int mods = (averageModsPerIteration >> 1) + 1 + rnd.nextInt(averageModsPerIteration); - while (mods > 0) - { - int v = rnd.nextInt(upperBound); - int rc = Math.max(0, Math.min(mods, maxRunLength) - 1); - int c = 1 + (rc <= 0 ? 0 : rnd.nextInt(rc)); - for (int j = 0 ; j < c ; j++) - { - buffer.put(v, v); - v++; - } - mods -= c; - } - Timer.Context ctxt; - ctxt = TREE_TIMER.time(); - canon.putAll(buffer); - ctxt.stop(); - ctxt = BTREE_TIMER.time(); - Object[] next = null; - while (next == null) - next = BTree.update(btree, ICMP, buffer.keySet(), true, SPORADIC_ABORT); - btree = next; - ctxt.stop(); - - if (!BTree.isWellFormed(btree, ICMP)) - { - System.out.println("ERROR: Not well formed"); - throw new AssertionError("Not well formed!"); - } - if (quickEquality) - testEqual("", BTree.slice(btree, true), canon.keySet().iterator()); - else - r.addAll(testAllSlices("RND", btree, new TreeSet<>(canon.keySet()))); - } - return r; - } - }); - MODIFY.execute(f); - return f; - } - - @Test - public void testSlicingAllSmallTrees() throws ExecutionException, InterruptedException - { - Object[] cur = BTree.empty(); - TreeSet canon = new TreeSet<>(); - // we set FAN_FACTOR to 4, so 128 items is four levels deep, three fully populated - for (int i = 0 ; i < 128 ; i++) - { - String id = String.format("[0..%d)", canon.size()); - System.out.println("Testing " + id); - Futures.allAsList(testAllSlices(id, cur, canon)).get(); - Object[] next = null; - while (next == null) - next = BTree.update(cur, ICMP, Arrays.asList(i), true, SPORADIC_ABORT); - cur = next; - canon.add(i); - } - } - - static final Comparator ICMP = new Comparator() - { - @Override - public int compare(Integer o1, Integer o2) - { - return Integer.compare(o1, o2); - } - }; - - private static List> testAllSlices(String id, Object[] btree, NavigableSet canon) - { - List> waitFor = new ArrayList<>(); - testAllSlices(id + " ASC", new BTreeSet<>(btree, ICMP), canon, true, waitFor); - testAllSlices(id + " DSC", new BTreeSet<>(btree, ICMP).descendingSet(), canon.descendingSet(), false, waitFor); - return waitFor; - } - - private static void testAllSlices(String id, NavigableSet btree, NavigableSet canon, boolean ascending, List> results) - { - testOneSlice(id, btree, canon, results); - for (Integer lb : range(canon.size(), Integer.MIN_VALUE, ascending)) - { - // test head/tail sets - testOneSlice(String.format("%s->[%d..)", id, lb), btree.headSet(lb, true), canon.headSet(lb, true), results); - testOneSlice(String.format("%s->(%d..)", id, lb), btree.headSet(lb, false), canon.headSet(lb, false), results); - testOneSlice(String.format("%s->(..%d]", id, lb), btree.tailSet(lb, true), canon.tailSet(lb, true), results); - testOneSlice(String.format("%s->(..%d]", id, lb), btree.tailSet(lb, false), canon.tailSet(lb, false), results); - for (Integer ub : range(canon.size(), lb, ascending)) - { - // test subsets - testOneSlice(String.format("%s->[%d..%d]", id, lb, ub), btree.subSet(lb, true, ub, true), canon.subSet(lb, true, ub, true), results); - testOneSlice(String.format("%s->(%d..%d]", id, lb, ub), btree.subSet(lb, false, ub, true), canon.subSet(lb, false, ub, true), results); - testOneSlice(String.format("%s->[%d..%d)", id, lb, ub), btree.subSet(lb, true, ub, false), canon.subSet(lb, true, ub, false), results); - testOneSlice(String.format("%s->(%d..%d)", id, lb, ub), btree.subSet(lb, false, ub, false), canon.subSet(lb, false, ub, false), results); - } - } - } - - private static void testOneSlice(final String id, final NavigableSet test, final NavigableSet canon, List> results) - { - ListenableFutureTask f = ListenableFutureTask.create(new Runnable() - { - - @Override - public void run() - { - test(id + " Count", test.size(), canon.size()); - testEqual(id, test.iterator(), canon.iterator()); - testEqual(id + "->DSCI", test.descendingIterator(), canon.descendingIterator()); - testEqual(id + "->DSCS", test.descendingSet().iterator(), canon.descendingSet().iterator()); - testEqual(id + "->DSCS->DSCI", test.descendingSet().descendingIterator(), canon.descendingSet().descendingIterator()); - } - }, null); - results.add(f); - COMPARE.execute(f); - } - - private static void test(String id, int test, int expect) - { - if (test != expect) - { - System.out.println(String.format("%s: Expected %d, Got %d", id, expect, test)); - } - } - - private static void testEqual(String id, Iterator btree, Iterator canon) - { - boolean equal = true; - while (btree.hasNext() && canon.hasNext()) - { - Object i = btree.next(); - Object j = canon.next(); - if (!i.equals(j)) - { - System.out.println(String.format("%s: Expected %d, Got %d", id, j, i)); - equal = false; - } - } - while (btree.hasNext()) - { - System.out.println(String.format("%s: Expected , Got %d", id, btree.next())); - equal = false; - } - while (canon.hasNext()) - { - System.out.println(String.format("%s: Expected %d, Got Nil", id, canon.next())); - equal = false; - } - if (!equal) - throw new AssertionError("Not equal"); - } - - // should only be called on sets that range from 0->N or N->0 - private static final Iterable range(final int size, final int from, final boolean ascending) - { - return new Iterable() - { - int cur; - int delta; - int end; - { - if (ascending) - { - end = size + 1; - cur = from == Integer.MIN_VALUE ? -1 : from; - delta = 1; - } - else - { - end = -2; - cur = from == Integer.MIN_VALUE ? size : from; - delta = -1; - } - } - @Override - public Iterator iterator() - { - return new Iterator() - { - @Override - public boolean hasNext() - { - return cur != end; - } - - @Override - public Integer next() - { - Integer r = cur; - cur += delta; - return r; - } - - @Override - public void remove() - { - throw new UnsupportedOperationException(); - } - }; - } - }; - } - - private static Object[] randomTree(int maxSize, Random random) - { - TreeSet build = new TreeSet<>(); - int size = random.nextInt(maxSize); - for (int i = 0 ; i < size ; i++) - { - build.add(random.nextInt()); - } - return BTree.build(build, ICMP, true, UpdateFunction.NoOp.instance()); - } - - private static Iterable randomSelection(Object[] iter, final Random rnd) - { - final float proportion = rnd.nextFloat(); - return Iterables.filter(new BTreeSet<>(iter, ICMP), new Predicate() - { - public boolean apply(Integer integer) - { - return rnd.nextFloat() < proportion; - } - }); - } - - private static Iterable randomMix(Object[] iter, final Random rnd) - { - final float proportion = rnd.nextFloat(); - return Iterables.transform(new BTreeSet<>(iter, ICMP), new Function() - { - long last = Integer.MIN_VALUE; - - public Integer apply(Integer v) - { - long last = this.last; - this.last = v; - if (rnd.nextFloat() < proportion) - return v; - return (int)((v - last) / 2); - } - }); - } - - private static final class RandomAbort implements UpdateFunction - { - final Random rnd; - final float chance; - private RandomAbort(Random rnd, float chance) - { - this.rnd = rnd; - this.chance = chance; - } - - public V apply(V replacing, V update) - { - return update; - } - - public boolean abortEarly() - { - return rnd.nextFloat() < chance; - } - - public void allocated(long heapSize) - { - - } - - public V apply(V v) - { - return v; - } - } -} diff --git a/test/conf/cassandra.yaml b/test/conf/cassandra.yaml index 3d3de84e73..cf76e7536c 100644 --- a/test/conf/cassandra.yaml +++ b/test/conf/cassandra.yaml @@ -3,7 +3,7 @@ # Consider the effects on 'o.a.c.i.s.LegacySSTableTest' before changing schemas in this file. # cluster_name: Test Cluster -memtable_allocation_type: offheap_objects +memtable_allocation_type: heap_buffers commitlog_sync: batch commitlog_sync_batch_window_in_ms: 1.0 commitlog_segment_size_in_mb: 5 diff --git a/test/conf/logback-test.xml b/test/conf/logback-test.xml index 8cb2d6f9c9..9facb836b5 100644 --- a/test/conf/logback-test.xml +++ b/test/conf/logback-test.xml @@ -68,7 +68,7 @@ - + diff --git a/test/data/corrupt-sstables/Keyspace1-Standard3-jb-1-CompressionInfo.db b/test/data/corrupt-sstables/Keyspace1-Standard3-jb-1-CompressionInfo.db deleted file mode 100644 index 44d2e59464aa223d34503618d8ff82199c1d7da0..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 43 fcmZSJ^@%cZ&d)6uuvuFik diff --git a/test/data/corrupt-sstables/Keyspace1-Standard3-jb-1-Data.db b/test/data/corrupt-sstables/Keyspace1-Standard3-jb-1-Data.db deleted file mode 100644 index f75c4e67b8b0ecc0031a80a4edbccde9e19fc614..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 133 zcmeZbWMG)gz?fG5|Ns9621W)!W)6lTmghdxn;1QK7%UkWlY|)<`5E*XQ_UGzbYqju z8JL6_n9LdEQ-FLMXQvZ+Fo8sK24jHytbIhbZlfpYU8+(amMDTJE@<=%mClObG2VGs=f!?gsK diff --git a/test/data/corrupt-sstables/Keyspace1-Standard3-jb-1-Summary.db b/test/data/corrupt-sstables/Keyspace1-Standard3-jb-1-Summary.db deleted file mode 100644 index 376ca9d364e19b4a0a1724305569cf8beb90daf3..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 71 ucmZQzU}#`qU|Tj^g~>RKLXBRHz`s|No2(jEO|5OCnNT6_M(aiBwlfq`DNM G)BymEOGCK; literal 0 HcmV?d00001 diff --git a/test/data/corrupt-sstables/la-1-big-Digest.adler32 b/test/data/corrupt-sstables/la-1-big-Digest.adler32 new file mode 100644 index 0000000000..e4472771e7 --- /dev/null +++ b/test/data/corrupt-sstables/la-1-big-Digest.adler32 @@ -0,0 +1 @@ +2370519993 \ No newline at end of file diff --git a/test/data/corrupt-sstables/la-1-big-Filter.db b/test/data/corrupt-sstables/la-1-big-Filter.db new file mode 100644 index 0000000000000000000000000000000000000000..709d2eadc137258de1990a87ae5537860e7e85e6 GIT binary patch literal 24 fcmZQzU|?lnU|>>6W@=+-VNqe|O7URmc)$z*AyWik literal 0 HcmV?d00001 diff --git a/test/data/corrupt-sstables/la-1-big-Index.db b/test/data/corrupt-sstables/la-1-big-Index.db new file mode 100644 index 0000000000000000000000000000000000000000..178221e3ed9e5a6306b4698269537e028d33e91e GIT binary patch literal 105 vcmZQzEMY(fjEPX52AF0{f^q{O+$t!y0>Vv(au-0jl~C>p2sZ`F{Q#l?4-f`H literal 0 HcmV?d00001 diff --git a/test/data/corrupt-sstables/Keyspace1-Standard3-jb-1-Statistics.db b/test/data/corrupt-sstables/la-1-big-Statistics.db similarity index 83% rename from test/data/corrupt-sstables/Keyspace1-Standard3-jb-1-Statistics.db rename to test/data/corrupt-sstables/la-1-big-Statistics.db index 07626156de61963cbce6b4e8a13ee06e0a268973..23b76acd5dea8a1311940d5a4c01dca12ecf3706 100644 GIT binary patch delta 361 zcmZ3cv{FTdfq{Vqh(Q3vW(49`AZ7yM4j^V`U=Z9o`G$a@v3*O`L-%ze)j&ZIQ2zh_ z|36+yM)rF<3MU-zVOnyqhi}K4*-Q-=elosld%?PA_Clr&!i)A^*fyK62Qx9U zZxoSd+`OENkF6eJ2%M}3ng+r!f&Cv~EQS;yi_!5uQ1Sp=j1SI&Fc=v)it}?*{StFi zfdZ&JunKml2KELBBbNbY8_-;kt(y5o>3WF;iOCtMddZ2!#ff<-MTvSTNqV`7Ma3D3 WIeL!8$(flUl?AEbAR<9I&~O0V7+!V& delta 117 zcmZ3fvP_AEfq`NAWC1Sm&Ei}VjEroH4Y=7h?_m#Snk>)7!N~}e1p&6r3S6@6$}G=) zroCG1!2qPEH!*tD|NsBrzNPA+`#O 0; - } -} - diff --git a/test/long/org/apache/cassandra/db/LongKeyspaceTest.java b/test/long/org/apache/cassandra/db/LongKeyspaceTest.java deleted file mode 100644 index fe22da849b..0000000000 --- a/test/long/org/apache/cassandra/db/LongKeyspaceTest.java +++ /dev/null @@ -1,82 +0,0 @@ -/** - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.cassandra.db; - -import org.junit.BeforeClass; -import org.junit.Test; - -import org.apache.cassandra.SchemaLoader; -import org.apache.cassandra.config.KSMetaData; -import org.apache.cassandra.exceptions.ConfigurationException; -import org.apache.cassandra.locator.SimpleStrategy; -import org.apache.cassandra.utils.WrappedRunnable; -import static org.apache.cassandra.Util.column; - -import org.apache.cassandra.Util; - -public class LongKeyspaceTest -{ - public static final String KEYSPACE1 = "LongKeyspaceTest"; - public static final String CF_STANDARD = "Standard1"; - - @BeforeClass - public static void defineSchema() throws ConfigurationException - { - SchemaLoader.prepareServer(); - SchemaLoader.createKeyspace(KEYSPACE1, - SimpleStrategy.class, - KSMetaData.optsWithRF(1), - SchemaLoader.standardCFMD(KEYSPACE1, CF_STANDARD)); - } - - @Test - public void testGetRowMultiColumn() throws Throwable - { - final Keyspace keyspace = Keyspace.open(KEYSPACE1); - final ColumnFamilyStore cfStore = keyspace.getColumnFamilyStore("Standard1"); - - for (int i = 1; i < 5000; i += 100) - { - Mutation rm = new Mutation(KEYSPACE1, Util.dk("key" + i).getKey()); - ColumnFamily cf = ArrayBackedSortedColumns.factory.create(KEYSPACE1, "Standard1"); - for (int j = 0; j < i; j++) - cf.addColumn(column("c" + j, "v" + j, 1L)); - rm.add(cf); - rm.applyUnsafe(); - } - - Runnable verify = new WrappedRunnable() - { - public void runMayThrow() throws Exception - { - ColumnFamily cf; - for (int i = 1; i < 5000; i += 100) - { - for (int j = 0; j < i; j++) - { - cf = cfStore.getColumnFamily(Util.namesQueryFilter(cfStore, Util.dk("key" + i), "c" + j)); - KeyspaceTest.assertColumns(cf, "c" + j); - } - } - - } - }; - KeyspaceTest.reTest(keyspace.getColumnFamilyStore("Standard1"), verify); - } -} diff --git a/test/long/org/apache/cassandra/db/commitlog/ComitLogStress.java b/test/long/org/apache/cassandra/db/commitlog/ComitLogStress.java deleted file mode 100644 index b4efd49d98..0000000000 --- a/test/long/org/apache/cassandra/db/commitlog/ComitLogStress.java +++ /dev/null @@ -1,97 +0,0 @@ -package org.apache.cassandra.db.commitlog; -/* - * - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - * - */ - - -import java.nio.ByteBuffer; -import java.util.concurrent.ArrayBlockingQueue; -import java.util.concurrent.ExecutorService; -import java.util.concurrent.Executors; -import java.util.concurrent.ScheduledExecutorService; -import java.util.concurrent.TimeUnit; -import java.util.concurrent.atomic.AtomicLong; - -import org.apache.cassandra.Util; -import org.apache.cassandra.concurrent.JMXEnabledThreadPoolExecutor; -import org.apache.cassandra.concurrent.NamedThreadFactory; -import org.apache.cassandra.db.Mutation; -import org.apache.cassandra.utils.ByteBufferUtil; -import org.apache.cassandra.utils.UUIDGen; - -public class ComitLogStress -{ - - public static final String format = "%s,%s,%s,%s,%s,%s"; - - public static void main(String[] args) throws Exception { - int NUM_THREADS = Runtime.getRuntime().availableProcessors(); - if (args.length >= 1) { - NUM_THREADS = Integer.parseInt(args[0]); - System.out.println("Setting num threads to: " + NUM_THREADS); - } - ExecutorService executor = new JMXEnabledThreadPoolExecutor(NUM_THREADS, NUM_THREADS, 60, - TimeUnit.SECONDS, new ArrayBlockingQueue(10 * NUM_THREADS), new NamedThreadFactory("Stress"), ""); - ScheduledExecutorService scheduled = Executors.newScheduledThreadPool(1); - - org.apache.cassandra.SchemaLoader.loadSchema(); - org.apache.cassandra.SchemaLoader.schemaDefinition(""); // leave def. blank to maintain old behaviour - final AtomicLong count = new AtomicLong(); - final long start = System.currentTimeMillis(); - System.out.println(String.format(format, "seconds", "max_mb", "allocated_mb", "free_mb", "diffrence", "count")); - scheduled.scheduleAtFixedRate(new Runnable() { - long lastUpdate = 0; - - public void run() { - Runtime runtime = Runtime.getRuntime(); - long maxMemory = mb(runtime.maxMemory()); - long allocatedMemory = mb(runtime.totalMemory()); - long freeMemory = mb(runtime.freeMemory()); - long temp = count.get(); - System.out.println(String.format(format, ((System.currentTimeMillis() - start) / 1000), - maxMemory, allocatedMemory, freeMemory, (temp - lastUpdate), lastUpdate)); - lastUpdate = temp; - } - }, 1, 1, TimeUnit.SECONDS); - - while (true) { - executor.execute(new CommitlogExecutor()); - count.incrementAndGet(); - } - } - - private static long mb(long maxMemory) { - return maxMemory / (1024 * 1024); - } - - static final String keyString = UUIDGen.getTimeUUID().toString(); - public static class CommitlogExecutor implements Runnable { - public void run() { - String ks = "Keyspace1"; - ByteBuffer key = ByteBufferUtil.bytes(keyString); - for (int i=0; i<100; ++i) { - Mutation mutation = new Mutation(ks, key); - mutation.add("Standard1", Util.cellname("name"), ByteBufferUtil.bytes("value" + i), - System.currentTimeMillis()); - CommitLog.instance.add(mutation); - } - } - } -} diff --git a/test/long/org/apache/cassandra/db/commitlog/CommitLogStressTest.java b/test/long/org/apache/cassandra/db/commitlog/CommitLogStressTest.java index 5897dec18e..9b4dde7bc7 100644 --- a/test/long/org/apache/cassandra/db/commitlog/CommitLogStressTest.java +++ b/test/long/org/apache/cassandra/db/commitlog/CommitLogStressTest.java @@ -30,6 +30,7 @@ import java.nio.ByteBuffer; import java.nio.charset.StandardCharsets; import java.util.ArrayList; import java.util.Collection; +import java.util.Iterator; import java.util.List; import java.util.Map; import java.util.Random; @@ -48,15 +49,20 @@ import org.junit.BeforeClass; import org.junit.Test; import org.apache.cassandra.SchemaLoader; import org.apache.cassandra.Util; +import org.apache.cassandra.UpdateBuilder; import org.apache.cassandra.config.Config.CommitLogSync; import org.apache.cassandra.config.DatabaseDescriptor; import org.apache.cassandra.config.ParameterizedClass; import org.apache.cassandra.config.Schema; -import org.apache.cassandra.db.Cell; -import org.apache.cassandra.db.ColumnFamily; -import org.apache.cassandra.db.ColumnSerializer; import org.apache.cassandra.db.Mutation; +import org.apache.cassandra.db.RowUpdateBuilder; +import org.apache.cassandra.db.rows.Cell; +import org.apache.cassandra.db.rows.Row; +import org.apache.cassandra.db.rows.SerializationHelper; +import org.apache.cassandra.db.partitions.PartitionUpdate; +import org.apache.cassandra.db.marshal.UTF8Type; import org.apache.cassandra.io.util.FastByteArrayInputStream; +import org.apache.cassandra.utils.FBUtilities; public class CommitLogStressTest { @@ -354,8 +360,7 @@ public class CommitLogStressTest } double time = (System.currentTimeMillis() - start) / 1000.0; double avg = (temp / time); - System.out - .println( + System.out.println( String.format("second %d mem max %.0fmb allocated %.0fmb free %.0fmb mutations %d since start %d avg %.3f content %.1fmb ondisk %.1fmb transfer %.3fmb", ((System.currentTimeMillis() - start) / 1000), mb(maxMemory), @@ -421,20 +426,20 @@ public class CommitLogStressTest { if (rl != null) rl.acquire(); - String ks = "Keyspace1"; ByteBuffer key = randomBytes(16, rand); - Mutation mutation = new Mutation(ks, key); + UpdateBuilder builder = UpdateBuilder.create(Schema.instance.getCFMetaData("Keyspace1", "Standard1"), Util.dk(key)); for (int ii = 0; ii < numCells; ii++) { int sz = randomSize ? rand.nextInt(cellSize) : cellSize; ByteBuffer bytes = randomBytes(sz, rand); - mutation.add("Standard1", Util.cellname("name" + ii), bytes, System.currentTimeMillis()); + builder.newRow("name" + ii).add("val", bytes); hash = hash(hash, bytes); ++cells; dataSize += sz; } - rp = commitLog.add(mutation); + + rp = commitLog.add(new Mutation(builder.build())); counter.incrementAndGet(); } } @@ -468,7 +473,7 @@ public class CommitLogStressTest { mutation = Mutation.serializer.deserialize(new DataInputStream(bufIn), desc.getMessagingVersion(), - ColumnSerializer.Flag.LOCAL); + SerializationHelper.Flag.LOCAL); } catch (IOException e) { @@ -476,18 +481,24 @@ public class CommitLogStressTest throw new AssertionError(e); } - for (ColumnFamily cf : mutation.getColumnFamilies()) + for (PartitionUpdate cf : mutation.getPartitionUpdates()) { - for (Cell c : cf.getSortedColumns()) + + Iterator rowIterator = cf.iterator(); + + while (rowIterator.hasNext()) { - if (new String(c.name().toByteBuffer().array(), StandardCharsets.UTF_8).startsWith("name")) + Row row = rowIterator.next(); + if (!(UTF8Type.instance.compose(row.clustering().get(0)).startsWith("name"))) + continue; + + for (Cell cell : row) { - hash = hash(hash, c.value()); + hash = hash(hash, cell.value()); ++cells; } } } } - } } diff --git a/test/long/org/apache/cassandra/db/compaction/LongCompactionsTest.java b/test/long/org/apache/cassandra/db/compaction/LongCompactionsTest.java index e6c8f56f5f..ca223ca31c 100644 --- a/test/long/org/apache/cassandra/db/compaction/LongCompactionsTest.java +++ b/test/long/org/apache/cassandra/db/compaction/LongCompactionsTest.java @@ -26,12 +26,14 @@ import org.junit.BeforeClass; import org.junit.Before; import org.junit.Test; +import org.apache.cassandra.UpdateBuilder; import org.apache.cassandra.SchemaLoader; import org.apache.cassandra.config.KSMetaData; import org.apache.cassandra.config.Schema; import org.apache.cassandra.Util; import org.apache.cassandra.exceptions.ConfigurationException; import org.apache.cassandra.db.*; +import org.apache.cassandra.db.partitions.*; import org.apache.cassandra.io.sstable.format.SSTableReader; import org.apache.cassandra.db.lifecycle.LifecycleTransaction; import org.apache.cassandra.io.sstable.SSTableUtils; @@ -93,7 +95,7 @@ public class LongCompactionsTest testCompaction(100, 800, 5); } - protected void testCompaction(int sstableCount, int rowsPerSSTable, int colsPerRow) throws Exception + protected void testCompaction(int sstableCount, int partitionsPerSSTable, int rowsPerPartition) throws Exception { CompactionManager.instance.disableAutoCompaction(); @@ -103,17 +105,16 @@ public class LongCompactionsTest ArrayList sstables = new ArrayList<>(); for (int k = 0; k < sstableCount; k++) { - SortedMap rows = new TreeMap(); - for (int j = 0; j < rowsPerSSTable; j++) + SortedMap rows = new TreeMap<>(); + for (int j = 0; j < partitionsPerSSTable; j++) { String key = String.valueOf(j); - Cell[] cols = new Cell[colsPerRow]; - for (int i = 0; i < colsPerRow; i++) - { - // last sstable has highest timestamps - cols[i] = Util.column(String.valueOf(i), String.valueOf(i), k); - } - rows.put(key, SSTableUtils.createCF(KEYSPACE1, CF_STANDARD, Long.MIN_VALUE, Integer.MIN_VALUE, cols)); + // last sstable has highest timestamps + UpdateBuilder builder = UpdateBuilder.create(store.metadata, String.valueOf(j)) + .withTimestamp(k); + for (int i = 0; i < rowsPerPartition; i++) + builder.newRow(String.valueOf(i)).add("val", String.valueOf(i)); + rows.put(key, builder.build()); } SSTableReader sstable = SSTableUtils.prepare().write(rows); sstables.add(sstable); @@ -133,8 +134,8 @@ public class LongCompactionsTest System.out.println(String.format("%s: sstables=%d rowsper=%d colsper=%d: %d ms", this.getClass().getName(), sstableCount, - rowsPerSSTable, - colsPerRow, + partitionsPerSSTable, + rowsPerPartition, TimeUnit.NANOSECONDS.toMillis(System.nanoTime() - start))); } @@ -157,23 +158,23 @@ public class LongCompactionsTest for (int j = 0; j < SSTABLES; j++) { for (int i = 0; i < ROWS_PER_SSTABLE; i++) { DecoratedKey key = Util.dk(String.valueOf(i % 2)); - Mutation rm = new Mutation(KEYSPACE1, key.getKey()); long timestamp = j * ROWS_PER_SSTABLE + i; - rm.add("Standard1", Util.cellname(String.valueOf(i / 2)), - ByteBufferUtil.EMPTY_BYTE_BUFFER, - timestamp); maxTimestampExpected = Math.max(timestamp, maxTimestampExpected); - rm.apply(); + UpdateBuilder.create(cfs.metadata, key) + .withTimestamp(timestamp) + .newRow(String.valueOf(i / 2)).add("val", ByteBufferUtil.EMPTY_BYTE_BUFFER) + .apply(); + inserted.add(key); } cfs.forceBlockingFlush(); CompactionsTest.assertMaxTimestamp(cfs, maxTimestampExpected); - assertEquals(inserted.toString(), inserted.size(), Util.getRangeSlice(cfs).size()); + + assertEquals(inserted.toString(), inserted.size(), Util.getAll(Util.cmd(cfs).build()).size()); } forceCompactions(cfs); - - assertEquals(inserted.size(), Util.getRangeSlice(cfs).size()); + assertEquals(inserted.toString(), inserted.size(), Util.getAll(Util.cmd(cfs).build()).size()); // make sure max timestamp of compacted sstables is recorded properly after compaction. CompactionsTest.assertMaxTimestamp(cfs, maxTimestampExpected); diff --git a/test/long/org/apache/cassandra/db/compaction/LongLeveledCompactionStrategyTest.java b/test/long/org/apache/cassandra/db/compaction/LongLeveledCompactionStrategyTest.java index cc8203ef24..fbee72aa03 100644 --- a/test/long/org/apache/cassandra/db/compaction/LongLeveledCompactionStrategyTest.java +++ b/test/long/org/apache/cassandra/db/compaction/LongLeveledCompactionStrategyTest.java @@ -27,6 +27,7 @@ import org.junit.Test; import org.apache.cassandra.SchemaLoader; import org.apache.cassandra.Util; +import org.apache.cassandra.UpdateBuilder; import org.apache.cassandra.config.KSMetaData; import org.apache.cassandra.db.*; import org.apache.cassandra.exceptions.ConfigurationException; @@ -75,11 +76,11 @@ public class LongLeveledCompactionStrategyTest for (int r = 0; r < rows; r++) { DecoratedKey key = Util.dk(String.valueOf(r)); - Mutation rm = new Mutation(ksname, key.getKey()); + UpdateBuilder builder = UpdateBuilder.create(store.metadata, key); for (int c = 0; c < columns; c++) - { - rm.add(cfname, Util.cellname("column" + c), value, 0); - } + builder.newRow("column" + c).add("val", value); + + Mutation rm = new Mutation(builder.build()); rm.apply(); store.forceBlockingFlush(); } diff --git a/test/unit/org/apache/cassandra/AbstractReadCommandBuilder.java b/test/unit/org/apache/cassandra/AbstractReadCommandBuilder.java new file mode 100644 index 0000000000..575036f1c9 --- /dev/null +++ b/test/unit/org/apache/cassandra/AbstractReadCommandBuilder.java @@ -0,0 +1,327 @@ +/* + * + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + * + */ +package org.apache.cassandra; + +import java.nio.ByteBuffer; +import java.util.*; + +import org.apache.cassandra.config.ColumnDefinition; +import org.apache.cassandra.cql3.*; +import org.apache.cassandra.db.*; +import org.apache.cassandra.db.rows.Row; +import org.apache.cassandra.db.filter.*; +import org.apache.cassandra.db.partitions.*; +import org.apache.cassandra.db.marshal.*; +import org.apache.cassandra.dht.*; +import org.apache.cassandra.service.StorageService; +import org.apache.cassandra.utils.FBUtilities; + +public abstract class AbstractReadCommandBuilder +{ + protected final ColumnFamilyStore cfs; + protected int nowInSeconds; + + private int cqlLimit = -1; + private int pagingLimit = -1; + private boolean reversed = false; + + private Set columns; + protected final RowFilter filter = RowFilter.create(); + + private Slice.Bound lowerClusteringBound; + private Slice.Bound upperClusteringBound; + + private NavigableSet clusterings; + + // Use Util.cmd() instead of this ctor directly + AbstractReadCommandBuilder(ColumnFamilyStore cfs) + { + this.cfs = cfs; + this.nowInSeconds = FBUtilities.nowInSeconds(); + } + + public AbstractReadCommandBuilder withNowInSeconds(int nowInSec) + { + this.nowInSeconds = nowInSec; + return this; + } + + public AbstractReadCommandBuilder fromIncl(Object... values) + { + assert lowerClusteringBound == null && clusterings == null; + this.lowerClusteringBound = Slice.Bound.create(cfs.metadata.comparator, true, true, values); + return this; + } + + public AbstractReadCommandBuilder fromExcl(Object... values) + { + assert lowerClusteringBound == null && clusterings == null; + this.lowerClusteringBound = Slice.Bound.create(cfs.metadata.comparator, true, false, values); + return this; + } + + public AbstractReadCommandBuilder toIncl(Object... values) + { + assert upperClusteringBound == null && clusterings == null; + this.upperClusteringBound = Slice.Bound.create(cfs.metadata.comparator, false, true, values); + return this; + } + + public AbstractReadCommandBuilder toExcl(Object... values) + { + assert upperClusteringBound == null && clusterings == null; + this.upperClusteringBound = Slice.Bound.create(cfs.metadata.comparator, false, false, values); + return this; + } + + public AbstractReadCommandBuilder includeRow(Object... values) + { + assert lowerClusteringBound == null && upperClusteringBound == null; + + if (this.clusterings == null) + this.clusterings = new TreeSet<>(cfs.metadata.comparator); + + this.clusterings.add(cfs.metadata.comparator.make(values)); + return this; + } + + public AbstractReadCommandBuilder reverse() + { + this.reversed = true; + return this; + } + + public AbstractReadCommandBuilder withLimit(int newLimit) + { + this.cqlLimit = newLimit; + return this; + } + + public AbstractReadCommandBuilder withPagingLimit(int newLimit) + { + this.pagingLimit = newLimit; + return this; + } + + public AbstractReadCommandBuilder columns(String... columns) + { + if (this.columns == null) + this.columns = new HashSet<>(); + + for (String column : columns) + this.columns.add(ColumnIdentifier.getInterned(column, true)); + return this; + } + + private ByteBuffer bb(Object value, AbstractType type) + { + return value instanceof ByteBuffer ? (ByteBuffer)value : ((AbstractType)type).decompose(value); + } + + private AbstractType forValues(AbstractType collectionType) + { + assert collectionType instanceof CollectionType; + CollectionType ct = (CollectionType)collectionType; + switch (ct.kind) + { + case LIST: + case MAP: + return ct.valueComparator(); + case SET: + return ct.nameComparator(); + } + throw new AssertionError(); + } + + private AbstractType forKeys(AbstractType collectionType) + { + assert collectionType instanceof CollectionType; + CollectionType ct = (CollectionType)collectionType; + switch (ct.kind) + { + case LIST: + case MAP: + return ct.nameComparator(); + } + throw new AssertionError(); + } + + public AbstractReadCommandBuilder filterOn(String column, Operator op, Object value) + { + ColumnDefinition def = cfs.metadata.getColumnDefinition(ColumnIdentifier.getInterned(column, true)); + assert def != null; + + AbstractType type = def.type; + if (op == Operator.CONTAINS) + type = forValues(type); + else if (op == Operator.CONTAINS_KEY) + type = forKeys(type); + + this.filter.add(def, op, bb(value, type)); + return this; + } + + protected ColumnFilter makeColumnFilter() + { + if (columns == null) + return ColumnFilter.all(cfs.metadata); + + ColumnFilter.Builder builder = ColumnFilter.allColumnsBuilder(cfs.metadata); + for (ColumnIdentifier column : columns) + builder.add(cfs.metadata.getColumnDefinition(column)); + return builder.build(); + } + + protected ClusteringIndexFilter makeFilter() + { + if (clusterings != null) + { + return new ClusteringIndexNamesFilter(clusterings, reversed); + } + else + { + Slice slice = Slice.make(lowerClusteringBound == null ? Slice.Bound.BOTTOM : lowerClusteringBound, + upperClusteringBound == null ? Slice.Bound.TOP : upperClusteringBound); + return new ClusteringIndexSliceFilter(Slices.with(cfs.metadata.comparator, slice), reversed); + } + } + + protected DataLimits makeLimits() + { + DataLimits limits = cqlLimit < 0 ? DataLimits.NONE : DataLimits.cqlLimits(cqlLimit); + if (pagingLimit >= 0) + limits = limits.forPaging(pagingLimit); + return limits; + } + + public Row getOnlyRow() + { + return Util.getOnlyRow(build()); + } + + public Row getOnlyRowUnfiltered() + { + return Util.getOnlyRowUnfiltered(build()); + } + + public FilteredPartition getOnlyPartition() + { + return Util.getOnlyPartition(build()); + } + + public Partition getOnlyPartitionUnfiltered() + { + return Util.getOnlyPartitionUnfiltered(build()); + } + + public abstract ReadCommand build(); + + public static class SinglePartitionBuilder extends AbstractReadCommandBuilder + { + private final DecoratedKey partitionKey; + + SinglePartitionBuilder(ColumnFamilyStore cfs, DecoratedKey key) + { + super(cfs); + this.partitionKey = key; + } + + @Override + public ReadCommand build() + { + return SinglePartitionReadCommand.create(cfs.metadata, nowInSeconds, makeColumnFilter(), filter, makeLimits(), partitionKey, makeFilter()); + } + } + + public static class PartitionRangeBuilder extends AbstractReadCommandBuilder + { + private DecoratedKey startKey; + private boolean startInclusive; + private DecoratedKey endKey; + private boolean endInclusive; + + PartitionRangeBuilder(ColumnFamilyStore cfs) + { + super(cfs); + } + + public PartitionRangeBuilder fromKeyIncl(Object... values) + { + assert startKey == null; + this.startInclusive = true; + this.startKey = Util.makeKey(cfs.metadata, values); + return this; + } + + public PartitionRangeBuilder fromKeyExcl(Object... values) + { + assert startKey == null; + this.startInclusive = false; + this.startKey = Util.makeKey(cfs.metadata, values); + return this; + } + + public PartitionRangeBuilder toKeyIncl(Object... values) + { + assert endKey == null; + this.endInclusive = true; + this.endKey = Util.makeKey(cfs.metadata, values); + return this; + } + + public PartitionRangeBuilder toKeyExcl(Object... values) + { + assert endKey == null; + this.endInclusive = false; + this.endKey = Util.makeKey(cfs.metadata, values); + return this; + } + + @Override + public ReadCommand build() + { + PartitionPosition start = startKey; + if (start == null) + { + start = StorageService.getPartitioner().getMinimumToken().maxKeyBound(); + startInclusive = false; + } + PartitionPosition end = endKey; + if (end == null) + { + end = StorageService.getPartitioner().getMinimumToken().maxKeyBound(); + endInclusive = true; + } + + AbstractBounds bounds; + if (startInclusive && endInclusive) + bounds = new Bounds(start, end); + else if (startInclusive && !endInclusive) + bounds = new IncludingExcludingBounds(start, end); + else if (!startInclusive && endInclusive) + bounds = new Range(start, end); + else + bounds = new ExcludingBounds(start, end); + + return new PartitionRangeReadCommand(cfs.metadata, nowInSeconds, makeColumnFilter(), filter, makeLimits(), new DataRange(bounds, makeFilter())); + } + } +} diff --git a/test/unit/org/apache/cassandra/EmbeddedServer.java b/test/unit/org/apache/cassandra/EmbeddedServer.java deleted file mode 100644 index 25754ea4be..0000000000 --- a/test/unit/org/apache/cassandra/EmbeddedServer.java +++ /dev/null @@ -1,83 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - */ -package org.apache.cassandra; - -import java.util.concurrent.ExecutorService; -import java.util.concurrent.Executors; -import java.util.concurrent.TimeUnit; - -import org.apache.cassandra.service.CassandraDaemon; -import org.junit.AfterClass; -import org.junit.BeforeClass; - -public class EmbeddedServer extends SchemaLoader -{ - protected static CassandraDaemon daemon = null; - - enum GatewayService - { - Thrift - } - - public static GatewayService getDaemonGatewayService() - { - return GatewayService.Thrift; - } - - static ExecutorService executor = Executors.newSingleThreadExecutor(); - - @BeforeClass - public static void startCassandra() - - { - executor.execute(new Runnable() - { - public void run() - { - switch (getDaemonGatewayService()) - { - case Thrift: - default: - daemon = new org.apache.cassandra.service.CassandraDaemon(); - } - daemon.activate(); - } - }); - try - { - TimeUnit.SECONDS.sleep(3); - } - catch (InterruptedException e) - { - throw new AssertionError(e); - } - } - - @AfterClass - public static void stopCassandra() throws Exception - { - if (daemon != null) - { - daemon.deactivate(); - } - executor.shutdown(); - executor.shutdownNow(); - } - -} diff --git a/test/unit/org/apache/cassandra/MockSchema.java b/test/unit/org/apache/cassandra/MockSchema.java index c71c98bb24..f0a849ba64 100644 --- a/test/unit/org/apache/cassandra/MockSchema.java +++ b/test/unit/org/apache/cassandra/MockSchema.java @@ -21,7 +21,7 @@ package org.apache.cassandra; import java.io.File; import java.io.IOException; import java.io.RandomAccessFile; -import java.util.Set; +import java.util.*; import java.util.concurrent.atomic.AtomicInteger; import com.google.common.collect.ImmutableMap; @@ -32,7 +32,6 @@ import org.apache.cassandra.config.CFMetaData; import org.apache.cassandra.config.DatabaseDescriptor; import org.apache.cassandra.config.KSMetaData; import org.apache.cassandra.db.*; -import org.apache.cassandra.db.composites.SimpleSparseCellNameType; import org.apache.cassandra.db.marshal.UTF8Type; import org.apache.cassandra.dht.Murmur3Partitioner; import org.apache.cassandra.io.sstable.Component; @@ -119,12 +118,13 @@ public class MockSchema throw new RuntimeException(e); } } + SerializationHeader header = SerializationHeader.make(cfs.metadata, Collections.EMPTY_LIST); StatsMetadata metadata = (StatsMetadata) new MetadataCollector(cfs.metadata.comparator) - .finalizeMetadata(Murmur3Partitioner.instance.getClass().getCanonicalName(), 0.01f, -1) + .finalizeMetadata(Murmur3Partitioner.instance.getClass().getCanonicalName(), 0.01f, -1, header) .get(MetadataType.STATS); SSTableReader reader = SSTableReader.internalOpen(descriptor, components, cfs.metadata, Murmur3Partitioner.instance, segmentedFile.sharedCopy(), segmentedFile.sharedCopy(), indexSummary.sharedCopy(), - new AlwaysPresentFilter(), 1L, metadata, SSTableReader.OpenReason.NORMAL); + new AlwaysPresentFilter(), 1L, metadata, SSTableReader.OpenReason.NORMAL, header); reader.first = reader.last = readerBounds(generation); if (!keepRef) reader.selfRef().release(); @@ -140,10 +140,11 @@ public class MockSchema private static CFMetaData newCFMetaData(String ksname, String cfname) { - CFMetaData metadata = new CFMetaData(ksname, - cfname, - ColumnFamilyType.Standard, - new SimpleSparseCellNameType(UTF8Type.instance)); + CFMetaData metadata = CFMetaData.Builder.create(ksname, cfname) + .addPartitionKey("key", UTF8Type.instance) + .addClusteringColumn("col", UTF8Type.instance) + .addRegularColumn("value", UTF8Type.instance) + .build(); metadata.caching(CachingOptions.NONE); return metadata; } diff --git a/test/unit/org/apache/cassandra/SchemaLoader.java b/test/unit/org/apache/cassandra/SchemaLoader.java index 46f4a9a1e7..6aca407763 100644 --- a/test/unit/org/apache/cassandra/SchemaLoader.java +++ b/test/unit/org/apache/cassandra/SchemaLoader.java @@ -19,7 +19,6 @@ package org.apache.cassandra; import java.io.File; import java.io.IOException; -import java.nio.ByteBuffer; import java.util.*; import org.junit.After; @@ -29,6 +28,7 @@ import org.slf4j.LoggerFactory; import org.apache.cassandra.cache.CachingOptions; import org.apache.cassandra.config.*; +import org.apache.cassandra.cql3.CQLTester; import org.apache.cassandra.cql3.ColumnIdentifier; import org.apache.cassandra.db.*; import org.apache.cassandra.db.commitlog.CommitLog; @@ -45,6 +45,7 @@ import org.apache.cassandra.locator.AbstractReplicationStrategy; import org.apache.cassandra.locator.SimpleStrategy; import org.apache.cassandra.service.MigrationManager; import org.apache.cassandra.utils.ByteBufferUtil; +import org.apache.cassandra.utils.FBUtilities; public class SchemaLoader { @@ -71,26 +72,7 @@ public class SchemaLoader public static void prepareServer() { - // Cleanup first - try - { - cleanupAndLeaveDirs(); - } - catch (IOException e) - { - logger.error("Failed to cleanup and recreate directories and files."); - throw new RuntimeException(e); - } - - Thread.setDefaultUncaughtExceptionHandler(new Thread.UncaughtExceptionHandler() - { - public void uncaughtException(Thread t, Throwable e) - { - logger.error("Fatal exception in thread " + t, e); - } - }); - - Keyspace.setInitialized(); + CQLTester.prepareServer(false); } public static void startGossiper() @@ -143,172 +125,169 @@ public class SchemaLoader // Keyspace 1 schema.add(KSMetaData.testMetadata(ks1, - simple, - opts_rf1, + simple, + opts_rf1, - // Column Families - standardCFMD(ks1, "Standard1").compactionStrategyOptions(compactionOptions), - standardCFMD(ks1, "Standard2"), - standardCFMD(ks1, "Standard3"), - standardCFMD(ks1, "Standard4"), - standardCFMD(ks1, "StandardGCGS0").gcGraceSeconds(0), - standardCFMD(ks1, "StandardLong1"), - standardCFMD(ks1, "StandardLong2"), - CFMetaData.denseCFMetaData(ks1, "ValuesWithQuotes", BytesType.instance).defaultValidator(UTF8Type.instance), - superCFMD(ks1, "Super1", LongType.instance), - superCFMD(ks1, "Super2", LongType.instance), - superCFMD(ks1, "Super3", LongType.instance), - superCFMD(ks1, "Super4", UTF8Type.instance), - superCFMD(ks1, "Super5", bytes), - superCFMD(ks1, "Super6", LexicalUUIDType.instance, UTF8Type.instance), - indexCFMD(ks1, "Indexed1", true), - indexCFMD(ks1, "Indexed2", false), - CFMetaData.denseCFMetaData(ks1, "StandardInteger1", IntegerType.instance), - CFMetaData.denseCFMetaData(ks1, "StandardLong3", IntegerType.instance), - CFMetaData.denseCFMetaData(ks1, "Counter1", bytes).defaultValidator(CounterColumnType.instance), - CFMetaData.denseCFMetaData(ks1, "SuperCounter1", bytes, bytes).defaultValidator(CounterColumnType.instance), - superCFMD(ks1, "SuperDirectGC", BytesType.instance).gcGraceSeconds(0), - jdbcSparseCFMD(ks1, "JdbcInteger", IntegerType.instance).addColumnDefinition(integerColumn(ks1, "JdbcInteger")), - jdbcSparseCFMD(ks1, "JdbcUtf8", UTF8Type.instance).addColumnDefinition(utf8Column(ks1, "JdbcUtf8")), - jdbcCFMD(ks1, "JdbcLong", LongType.instance), - jdbcCFMD(ks1, "JdbcBytes", bytes), - jdbcCFMD(ks1, "JdbcAscii", AsciiType.instance), - CFMetaData.denseCFMetaData(ks1, "StandardComposite", composite), - CFMetaData.denseCFMetaData(ks1, "StandardComposite2", compositeMaxMin), - CFMetaData.denseCFMetaData(ks1, "StandardDynamicComposite", dynamicComposite), - standardCFMD(ks1, "StandardLeveled") - .compactionStrategyClass(LeveledCompactionStrategy.class) - .compactionStrategyOptions(leveledOptions), - standardCFMD(ks1, "legacyleveled") - .compactionStrategyClass(LeveledCompactionStrategy.class) - .compactionStrategyOptions(leveledOptions), - standardCFMD(ks1, "StandardLowIndexInterval").minIndexInterval(8) - .maxIndexInterval(256) - .caching(CachingOptions.NONE), - standardCFMD(ks1, "UUIDKeys").keyValidator(UUIDType.instance), - CFMetaData.denseCFMetaData(ks1, "MixedTypes", LongType.instance).keyValidator(UUIDType.instance).defaultValidator(BooleanType.instance), - CFMetaData.denseCFMetaData(ks1, "MixedTypesComposite", composite).keyValidator(composite).defaultValidator(BooleanType.instance), - standardCFMD(ks1, "AsciiKeys").keyValidator(AsciiType.instance) + // Column Families + standardCFMD(ks1, "Standard1").compactionStrategyOptions(compactionOptions), + standardCFMD(ks1, "Standard2"), + standardCFMD(ks1, "Standard3"), + standardCFMD(ks1, "Standard4"), + standardCFMD(ks1, "StandardGCGS0").gcGraceSeconds(0), + standardCFMD(ks1, "StandardLong1"), + standardCFMD(ks1, "StandardLong2"), + //CFMetaData.Builder.create(ks1, "ValuesWithQuotes").build(), + superCFMD(ks1, "Super1", LongType.instance), + superCFMD(ks1, "Super2", LongType.instance), + superCFMD(ks1, "Super3", LongType.instance), + superCFMD(ks1, "Super4", UTF8Type.instance), + superCFMD(ks1, "Super5", bytes), + superCFMD(ks1, "Super6", LexicalUUIDType.instance, UTF8Type.instance), + keysIndexCFMD(ks1, "Indexed1", true), + keysIndexCFMD(ks1, "Indexed2", false), + //CFMetaData.Builder.create(ks1, "StandardInteger1").withColumnNameComparator(IntegerType.instance).build(), + //CFMetaData.Builder.create(ks1, "StandardLong3").withColumnNameComparator(IntegerType.instance).build(), + //CFMetaData.Builder.create(ks1, "Counter1", false, false, true).build(), + //CFMetaData.Builder.create(ks1, "SuperCounter1", false, false, true, true).build(), + superCFMD(ks1, "SuperDirectGC", BytesType.instance).gcGraceSeconds(0), + jdbcCFMD(ks1, "JdbcInteger", IntegerType.instance).addColumnDefinition(integerColumn(ks1, "JdbcInteger")), + jdbcCFMD(ks1, "JdbcUtf8", UTF8Type.instance).addColumnDefinition(utf8Column(ks1, "JdbcUtf8")), + jdbcCFMD(ks1, "JdbcLong", LongType.instance), + jdbcCFMD(ks1, "JdbcBytes", bytes), + jdbcCFMD(ks1, "JdbcAscii", AsciiType.instance), + //CFMetaData.Builder.create(ks1, "StandardComposite", false, true, false).withColumnNameComparator(composite).build(), + //CFMetaData.Builder.create(ks1, "StandardComposite2", false, true, false).withColumnNameComparator(compositeMaxMin).build(), + //CFMetaData.Builder.create(ks1, "StandardDynamicComposite", false, true, false).withColumnNameComparator(dynamicComposite).build(), + standardCFMD(ks1, "StandardLeveled") + .compactionStrategyClass(LeveledCompactionStrategy.class) + .compactionStrategyOptions(leveledOptions), + standardCFMD(ks1, "legacyleveled") + .compactionStrategyClass(LeveledCompactionStrategy.class) + .compactionStrategyOptions(leveledOptions), + standardCFMD(ks1, "StandardLowIndexInterval").minIndexInterval(8) + .maxIndexInterval(256) + .caching(CachingOptions.NONE) + //CFMetaData.Builder.create(ks1, "UUIDKeys").addPartitionKey("key",UUIDType.instance).build(), + //CFMetaData.Builder.create(ks1, "MixedTypes").withColumnNameComparator(LongType.instance).addPartitionKey("key", UUIDType.instance).build(), + //CFMetaData.Builder.create(ks1, "MixedTypesComposite", false, true, false).withColumnNameComparator(composite).addPartitionKey("key", composite).build(), + //CFMetaData.Builder.create(ks1, "AsciiKeys").addPartitionKey("key", AsciiType.instance).build() )); // Keyspace 2 schema.add(KSMetaData.testMetadata(ks2, - simple, - opts_rf1, + simple, + opts_rf1, - // Column Families - standardCFMD(ks2, "Standard1"), - standardCFMD(ks2, "Standard3"), - superCFMD(ks2, "Super3", bytes), - superCFMD(ks2, "Super4", TimeUUIDType.instance), - indexCFMD(ks2, "Indexed1", true), - compositeIndexCFMD(ks2, "Indexed2", true), - compositeIndexCFMD(ks2, "Indexed3", true).gcGraceSeconds(0))); + // Column Families + standardCFMD(ks2, "Standard1"), + standardCFMD(ks2, "Standard3"), + superCFMD(ks2, "Super3", bytes), + superCFMD(ks2, "Super4", TimeUUIDType.instance), + keysIndexCFMD(ks2, "Indexed1", true), + compositeIndexCFMD(ks2, "Indexed2", true), + compositeIndexCFMD(ks2, "Indexed3", true).gcGraceSeconds(0))); // Keyspace 3 schema.add(KSMetaData.testMetadata(ks3, - simple, - opts_rf5, + simple, + opts_rf5, - // Column Families - standardCFMD(ks3, "Standard1"), - indexCFMD(ks3, "Indexed1", true))); + // Column Families + standardCFMD(ks3, "Standard1"), + keysIndexCFMD(ks3, "Indexed1", true))); // Keyspace 4 schema.add(KSMetaData.testMetadata(ks4, - simple, - opts_rf3, + simple, + opts_rf3, - // Column Families - standardCFMD(ks4, "Standard1"), - standardCFMD(ks4, "Standard3"), - superCFMD(ks4, "Super3", bytes), - superCFMD(ks4, "Super4", TimeUUIDType.instance), - CFMetaData.denseCFMetaData(ks4, "Super5", TimeUUIDType.instance, bytes))); + // Column Families + standardCFMD(ks4, "Standard1"), + standardCFMD(ks4, "Standard3"), + superCFMD(ks4, "Super3", bytes), + superCFMD(ks4, "Super4", TimeUUIDType.instance), + superCFMD(ks4, "Super5", TimeUUIDType.instance, BytesType.instance))); // Keyspace 5 schema.add(KSMetaData.testMetadata(ks5, - simple, - opts_rf2, - standardCFMD(ks5, "Standard1"), - standardCFMD(ks5, "Counter1") - .defaultValidator(CounterColumnType.instance))); + simple, + opts_rf2, + standardCFMD(ks5, "Standard1"))); // Keyspace 6 schema.add(KSMetaData.testMetadata(ks6, - simple, - opts_rf1, - indexCFMD(ks6, "Indexed1", true))); + simple, + opts_rf1, + keysIndexCFMD(ks6, "Indexed1", true))); // KeyCacheSpace schema.add(KSMetaData.testMetadata(ks_kcs, - simple, - opts_rf1, - standardCFMD(ks_kcs, "Standard1"), - standardCFMD(ks_kcs, "Standard2"), - standardCFMD(ks_kcs, "Standard3"))); + simple, + opts_rf1, + standardCFMD(ks_kcs, "Standard1"), + standardCFMD(ks_kcs, "Standard2"), + standardCFMD(ks_kcs, "Standard3"))); // RowCacheSpace schema.add(KSMetaData.testMetadata(ks_rcs, - simple, - opts_rf1, - standardCFMD(ks_rcs, "CFWithoutCache").caching(CachingOptions.NONE), - standardCFMD(ks_rcs, "CachedCF").caching(CachingOptions.ALL), - standardCFMD(ks_rcs, "CachedIntCF"). - defaultValidator(IntegerType.instance). - caching(new CachingOptions(new CachingOptions.KeyCache(CachingOptions.KeyCache.Type.ALL), - new CachingOptions.RowCache(CachingOptions.RowCache.Type.HEAD, 100))))); + simple, + opts_rf1, + standardCFMD(ks_rcs, "CFWithoutCache").caching(CachingOptions.NONE), + standardCFMD(ks_rcs, "CachedCF").caching(CachingOptions.ALL), + standardCFMD(ks_rcs, "CachedIntCF"). + caching(new CachingOptions(new CachingOptions.KeyCache(CachingOptions.KeyCache.Type.ALL), + new CachingOptions.RowCache(CachingOptions.RowCache.Type.HEAD, 100))))); // CounterCacheSpace - schema.add(KSMetaData.testMetadata(ks_ccs, - simple, - opts_rf1, - standardCFMD(ks_ccs, "Counter1").defaultValidator(CounterColumnType.instance), - standardCFMD(ks_ccs, "Counter2").defaultValidator(CounterColumnType.instance))); + /*schema.add(KSMetaData.testMetadata(ks_ccs, + simple, + opts_rf1, + CFMetaData.Builder.create(ks_ccs, "Counter1", false, false, true).build(), + CFMetaData.Builder.create(ks_ccs, "Counter1", false, false, true).build()));*/ schema.add(KSMetaData.testMetadataNotDurable(ks_nocommit, - simple, - opts_rf1, - standardCFMD(ks_nocommit, "Standard1"))); + simple, + opts_rf1, + standardCFMD(ks_nocommit, "Standard1"))); // PerRowSecondaryIndexTest schema.add(KSMetaData.testMetadata(ks_prsi, - simple, - opts_rf1, - perRowIndexedCFMD(ks_prsi, "Indexed1"))); + simple, + opts_rf1, + perRowIndexedCFMD(ks_prsi, "Indexed1"))); // CQLKeyspace schema.add(KSMetaData.testMetadata(ks_cql, - simple, - opts_rf1, + simple, + opts_rf1, - // Column Families - CFMetaData.compile("CREATE TABLE table1 (" - + "k int PRIMARY KEY," - + "v1 text," - + "v2 int" - + ")", ks_cql), + // Column Families + CFMetaData.compile("CREATE TABLE table1 (" + + "k int PRIMARY KEY," + + "v1 text," + + "v2 int" + + ")", ks_cql), - CFMetaData.compile("CREATE TABLE table2 (" - + "k text," - + "c text," - + "v text," - + "PRIMARY KEY (k, c))", ks_cql), - CFMetaData.compile("CREATE TABLE foo (" - + "bar text, " - + "baz text, " - + "qux text, " - + "PRIMARY KEY(bar, baz) ) " - + "WITH COMPACT STORAGE", ks_cql), - CFMetaData.compile("CREATE TABLE foofoo (" - + "bar text, " - + "baz text, " - + "qux text, " - + "quz text, " - + "foo text, " - + "PRIMARY KEY((bar, baz), qux, quz) ) " - + "WITH COMPACT STORAGE", ks_cql) - )); + CFMetaData.compile("CREATE TABLE table2 (" + + "k text," + + "c text," + + "v text," + + "PRIMARY KEY (k, c))", ks_cql), + CFMetaData.compile("CREATE TABLE foo (" + + "bar text, " + + "baz text, " + + "qux text, " + + "PRIMARY KEY(bar, baz) ) " + + "WITH COMPACT STORAGE", ks_cql), + CFMetaData.compile("CREATE TABLE foofoo (" + + "bar text, " + + "baz text, " + + "qux text, " + + "quz text, " + + "foo text, " + + "PRIMARY KEY((bar, baz), qux, quz) ) " + + "WITH COMPACT STORAGE", ks_cql) + )); if (Boolean.parseBoolean(System.getProperty("cassandra.test.compression", "false"))) @@ -344,7 +323,7 @@ public class SchemaLoader { return new ColumnDefinition(ksName, cfName, - new ColumnIdentifier(IntegerType.instance.fromString("42"), IntegerType.instance), + ColumnIdentifier.getInterned(IntegerType.instance.fromString("42"), IntegerType.instance), UTF8Type.instance, null, null, @@ -357,7 +336,7 @@ public class SchemaLoader { return new ColumnDefinition(ksName, cfName, - new ColumnIdentifier("fortytwo", true), + ColumnIdentifier.getInterned("fortytwo", true), UTF8Type.instance, null, null, @@ -372,10 +351,11 @@ public class SchemaLoader SecondaryIndex.CUSTOM_INDEX_OPTION_NAME, PerRowSecondaryIndexTest.TestIndex.class.getName()); - CFMetaData cfm = CFMetaData.sparseCFMetaData(ksName, cfName, AsciiType.instance).keyValidator(AsciiType.instance); + CFMetaData cfm = CFMetaData.Builder.create(ksName, cfName) + .addPartitionKey("key", AsciiType.instance) + .build(); - ByteBuffer cName = ByteBufferUtil.bytes("indexed"); - return cfm.addOrReplaceColumnDefinition(ColumnDefinition.regularDef(cfm, cName, AsciiType.instance, null) + return cfm.addOrReplaceColumnDefinition(ColumnDefinition.regularDef(ksName, cfName, "indexed", AsciiType.instance, null) .setIndex("indexe1", IndexType.CUSTOM, indexOptions)); } @@ -390,57 +370,128 @@ public class SchemaLoader } } + public static CFMetaData counterCFMD(String ksName, String cfName) + { + return CFMetaData.Builder.create(ksName, cfName, false, true, true) + .addPartitionKey("key", AsciiType.instance) + .addClusteringColumn("name", AsciiType.instance) + .addRegularColumn("val", CounterColumnType.instance) + .addRegularColumn("val2", CounterColumnType.instance) + .build() + .compressionParameters(getCompressionParameters()); + } + public static CFMetaData standardCFMD(String ksName, String cfName) { - return CFMetaData.denseCFMetaData(ksName, cfName, BytesType.instance).compressionParameters(getCompressionParameters()); + return standardCFMD(ksName, cfName, 1, AsciiType.instance); } - public static CFMetaData standardCFMD(String ksName, String cfName, AbstractType comparator) + public static CFMetaData standardCFMD(String ksName, String cfName, int columnCount, AbstractType keyType) { - return CFMetaData.denseCFMetaData(ksName, cfName, comparator).compressionParameters(getCompressionParameters()); + return standardCFMD(ksName, cfName, columnCount, keyType, AsciiType.instance); } + public static CFMetaData standardCFMD(String ksName, String cfName, int columnCount, AbstractType keyType, AbstractType valType) + { + return standardCFMD(ksName, cfName, columnCount, keyType, valType, AsciiType.instance); + } + + public static CFMetaData standardCFMD(String ksName, String cfName, int columnCount, AbstractType keyType, AbstractType valType, AbstractType clusteringType) + { + CFMetaData.Builder builder = CFMetaData.Builder.create(ksName, cfName) + .addPartitionKey("key", keyType) + .addClusteringColumn("name", clusteringType) + .addRegularColumn("val", valType); + + for (int i = 0; i < columnCount; i++) + builder.addRegularColumn("val" + i, AsciiType.instance); + + return builder.build() + .compressionParameters(getCompressionParameters()); + } + + public static CFMetaData denseCFMD(String ksName, String cfName) + { + return denseCFMD(ksName, cfName, AsciiType.instance); + } + public static CFMetaData denseCFMD(String ksName, String cfName, AbstractType cc) + { + return denseCFMD(ksName, cfName, cc, null); + } + public static CFMetaData denseCFMD(String ksName, String cfName, AbstractType cc, AbstractType subcc) + { + AbstractType comp = cc; + if (subcc != null) + comp = CompositeType.getInstance(Arrays.asList(new AbstractType[]{cc, subcc})); + + return CFMetaData.Builder.createDense(ksName, cfName, subcc != null, false) + .addPartitionKey("key", AsciiType.instance) + .addClusteringColumn("cols", comp) + .addRegularColumn("val", AsciiType.instance) + .build() + .compressionParameters(getCompressionParameters()); + } + + // TODO: Fix superCFMD failing on legacy table creation. Seems to be applying composite comparator to partition key public static CFMetaData superCFMD(String ksName, String cfName, AbstractType subcc) { - return superCFMD(ksName, cfName, BytesType.instance, subcc).compressionParameters(getCompressionParameters()); + return superCFMD(ksName, cfName, BytesType.instance, subcc); } - public static CFMetaData superCFMD(String ksName, String cfName, AbstractType cc, AbstractType subcc) { - return CFMetaData.denseCFMetaData(ksName, cfName, cc, subcc).compressionParameters(getCompressionParameters()); + return superCFMD(ksName, cfName, "cols", cc, subcc); } - - public static CFMetaData indexCFMD(String ksName, String cfName, final Boolean withIdxType) throws ConfigurationException + public static CFMetaData superCFMD(String ksName, String cfName, String ccName, AbstractType cc, AbstractType subcc) { - CFMetaData cfm = CFMetaData.sparseCFMetaData(ksName, cfName, BytesType.instance).keyValidator(AsciiType.instance); + //This is busted +// return CFMetaData.Builder.createSuper(ksName, cfName, false) +// .addPartitionKey("0", BytesType.instance) +// .addClusteringColumn("1", cc) +// .addClusteringColumn("2", subcc) +// .addRegularColumn("3", AsciiType.instance) +// .build(); + return standardCFMD(ksName, cfName); - ByteBuffer cName = ByteBufferUtil.bytes("birthdate"); - IndexType keys = withIdxType ? IndexType.KEYS : null; - return cfm.addColumnDefinition(ColumnDefinition.regularDef(cfm, cName, LongType.instance, null) - .setIndex(withIdxType ? ByteBufferUtil.bytesToHex(cName) : null, keys, null)) - .compressionParameters(getCompressionParameters()); } - - public static CFMetaData compositeIndexCFMD(String ksName, String cfName, final Boolean withIdxType) throws ConfigurationException + public static CFMetaData compositeIndexCFMD(String ksName, String cfName, boolean withIndex) throws ConfigurationException { - final CompositeType composite = CompositeType.getInstance(Arrays.asList(new AbstractType[]{UTF8Type.instance, UTF8Type.instance})); - CFMetaData cfm = CFMetaData.sparseCFMetaData(ksName, cfName, composite); + // the withIndex flag exists to allow tests index creation + // on existing columns + CFMetaData cfm = CFMetaData.Builder.create(ksName, cfName) + .addPartitionKey("key", AsciiType.instance) + .addClusteringColumn("c1", AsciiType.instance) + .addRegularColumn("birthdate", LongType.instance) + .addRegularColumn("notbirthdate", LongType.instance) + .build(); - ByteBuffer cName = ByteBufferUtil.bytes("col1"); - IndexType idxType = withIdxType ? IndexType.COMPOSITES : null; - return cfm.addColumnDefinition(ColumnDefinition.regularDef(cfm, cName, UTF8Type.instance, 1) - .setIndex(withIdxType ? "col1_idx" : null, idxType, Collections.emptyMap())) - .compressionParameters(getCompressionParameters()); + if (withIndex) + cfm.getColumnDefinition(new ColumnIdentifier("birthdate", true)) + .setIndex("birthdate_key_index", IndexType.COMPOSITES, Collections.EMPTY_MAP); + + return cfm.compressionParameters(getCompressionParameters()); + } + public static CFMetaData keysIndexCFMD(String ksName, String cfName, boolean withIndex) throws ConfigurationException + { + CFMetaData cfm = CFMetaData.Builder.createDense(ksName, cfName, false, false) + .addPartitionKey("key", AsciiType.instance) + .addClusteringColumn("c1", AsciiType.instance) + .addStaticColumn("birthdate", LongType.instance) + .addStaticColumn("notbirthdate", LongType.instance) + .addRegularColumn("value", LongType.instance) + .build(); + + if (withIndex) + cfm.getColumnDefinition(new ColumnIdentifier("birthdate", true)) + .setIndex("birthdate_composite_index", IndexType.KEYS, Collections.EMPTY_MAP); + + return cfm.compressionParameters(getCompressionParameters()); } - private static CFMetaData jdbcCFMD(String ksName, String cfName, AbstractType comp) + public static CFMetaData jdbcCFMD(String ksName, String cfName, AbstractType comp) { - return CFMetaData.denseCFMetaData(ksName, cfName, comp).defaultValidator(comp).compressionParameters(getCompressionParameters()); - } - - public static CFMetaData jdbcSparseCFMD(String ksName, String cfName, AbstractType comp) - { - return CFMetaData.sparseCFMetaData(ksName, cfName, comp).defaultValidator(comp).compressionParameters(getCompressionParameters()); + return CFMetaData.Builder.create(ksName, cfName).addPartitionKey("key", BytesType.instance) + .build() + .compressionParameters(getCompressionParameters()); } public static CompressionParameters getCompressionParameters() @@ -503,23 +554,13 @@ public class SchemaLoader public static void insertData(String keyspace, String columnFamily, int offset, int numberOfRows) { - for (int i = offset; i < offset + numberOfRows; i++) - { - ByteBuffer key = ByteBufferUtil.bytes("key" + i); - Mutation mutation = new Mutation(keyspace, key); - mutation.add(columnFamily, Util.cellname("col" + i), ByteBufferUtil.bytes("val" + i), System.currentTimeMillis()); - mutation.applyUnsafe(); - } - } + CFMetaData cfm = Schema.instance.getCFMetaData(keyspace, columnFamily); - /* usually used to populate the cache */ - public static void readData(String keyspace, String columnFamily, int offset, int numberOfRows) - { - ColumnFamilyStore store = Keyspace.open(keyspace).getColumnFamilyStore(columnFamily); for (int i = offset; i < offset + numberOfRows; i++) { - DecoratedKey key = Util.dk("key" + i); - store.getColumnFamily(Util.namesQueryFilter(store, key, "col" + i)); + RowUpdateBuilder builder = new RowUpdateBuilder(cfm, FBUtilities.timestampMicros(), ByteBufferUtil.bytes("key"+i)); + builder.clustering(ByteBufferUtil.bytes("col"+ i)).add("val", ByteBufferUtil.bytes("val" + i)); + builder.build().apply(); } } diff --git a/test/unit/org/apache/cassandra/UpdateBuilder.java b/test/unit/org/apache/cassandra/UpdateBuilder.java new file mode 100644 index 0000000000..dc6b8593a1 --- /dev/null +++ b/test/unit/org/apache/cassandra/UpdateBuilder.java @@ -0,0 +1,119 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.cassandra; + +import java.nio.ByteBuffer; + +import org.apache.cassandra.config.CFMetaData; +import org.apache.cassandra.db.*; +import org.apache.cassandra.db.partitions.*; +import org.apache.cassandra.utils.FBUtilities; +import org.apache.cassandra.service.StorageService; + + +/** + * Convenience object to create updates to a single partition. + * + * This is not unlike RowUpdateBuilder except that it allows to create update to multiple rows more easily. + * It is also aimed at unit tests so favor convenience over efficiency. + */ +public class UpdateBuilder +{ + private final PartitionUpdate update; + private RowUpdateBuilder currentRow; + private long timestamp = FBUtilities.timestampMicros(); + + private UpdateBuilder(CFMetaData metadata, DecoratedKey partitionKey) + { + this.update = new PartitionUpdate(metadata, partitionKey, metadata.partitionColumns(), 4); + } + + public static UpdateBuilder create(CFMetaData metadata, Object... partitionKey) + { + return new UpdateBuilder(metadata, makeKey(metadata, partitionKey)); + } + + public UpdateBuilder withTimestamp(long timestamp) + { + this.timestamp = timestamp; + return this; + } + + public UpdateBuilder newRow(Object... clustering) + { + maybeBuildCurrentRow(); + currentRow = new RowUpdateBuilder(update, timestamp, 0); + if (clustering.length > 0) + currentRow.clustering(clustering); + return this; + } + + public UpdateBuilder add(String column, Object value) + { + assert currentRow != null; + currentRow.add(column, value); + return this; + } + + public PartitionUpdate build() + { + maybeBuildCurrentRow(); + return update; + } + + public IMutation makeMutation() + { + Mutation m = new Mutation(build()); + return update.metadata().isCounter + ? new CounterMutation(m, ConsistencyLevel.ONE) + : m; + } + + public void apply() + { + Mutation m = new Mutation(build()); + if (update.metadata().isCounter) + new CounterMutation(m, ConsistencyLevel.ONE).apply(); + else + m.apply(); + } + + public void applyUnsafe() + { + assert !update.metadata().isCounter : "Counters have currently no applyUnsafe() option"; + new Mutation(build()).applyUnsafe(); + } + + private void maybeBuildCurrentRow() + { + if (currentRow != null) + { + currentRow.build(); + currentRow = null; + } + } + + private static DecoratedKey makeKey(CFMetaData metadata, Object[] partitionKey) + { + if (partitionKey.length == 1 && partitionKey[0] instanceof DecoratedKey) + return (DecoratedKey)partitionKey[0]; + + ByteBuffer key = CFMetaData.serializePartitionKey(metadata.getKeyValidatorAsClusteringComparator().make(partitionKey)); + return StorageService.getPartitioner().decorateKey(key); + } +} diff --git a/test/unit/org/apache/cassandra/Util.java b/test/unit/org/apache/cassandra/Util.java index 2d59abb281..423b3c0bd0 100644 --- a/test/unit/org/apache/cassandra/Util.java +++ b/test/unit/org/apache/cassandra/Util.java @@ -3,8 +3,7 @@ package org.apache.cassandra; * * Licensed to the Apache Software Foundation (ASF) under one * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file + * distributed with this work for additional information * regarding copyright ownership. The ASF licenses this file * to you under the Apache License, Version 2.0 (the * "License"); you may not use this file except in compliance * with the License. You may obtain a copy of the License at @@ -20,35 +19,37 @@ package org.apache.cassandra; * */ -import java.io.*; +import java.io.EOFException; import java.net.InetAddress; import java.net.UnknownHostException; import java.nio.ByteBuffer; -import java.nio.channels.FileChannel; import java.util.*; import java.util.concurrent.Callable; import java.util.concurrent.Future; +import java.util.concurrent.atomic.AtomicBoolean; import java.util.function.Supplier; -import com.google.common.collect.ImmutableMap; -import com.google.common.collect.ImmutableSet; +import com.google.common.base.Function; +import com.google.common.base.Preconditions; +import com.google.common.collect.Iterators; +import org.apache.commons.lang3.StringUtils; -import org.apache.cassandra.cache.CachingOptions; import org.apache.cassandra.config.CFMetaData; -import org.apache.cassandra.config.KSMetaData; +import org.apache.cassandra.config.ColumnDefinition; +import org.apache.cassandra.cql3.ColumnIdentifier; +import org.apache.cassandra.cql3.Operator; import org.apache.cassandra.db.*; -import org.apache.cassandra.db.composites.*; +import org.apache.cassandra.db.Slice.Bound; +import org.apache.cassandra.db.rows.*; import org.apache.cassandra.db.compaction.AbstractCompactionTask; import org.apache.cassandra.db.compaction.CompactionManager; -import org.apache.cassandra.db.columniterator.IdentityQueryFilter; -import org.apache.cassandra.db.filter.IDiskAtomFilter; -import org.apache.cassandra.db.filter.QueryFilter; -import org.apache.cassandra.db.filter.SliceQueryFilter; -import org.apache.cassandra.db.filter.NamesQueryFilter; +import org.apache.cassandra.db.filter.*; import org.apache.cassandra.db.marshal.AbstractType; -import org.apache.cassandra.db.marshal.UTF8Type; +import org.apache.cassandra.db.partitions.*; import org.apache.cassandra.dht.*; import org.apache.cassandra.dht.RandomPartitioner.BigIntegerToken; +import org.apache.cassandra.dht.Range; +import org.apache.cassandra.dht.Token; import org.apache.cassandra.gms.ApplicationState; import org.apache.cassandra.gms.Gossiper; import org.apache.cassandra.gms.VersionedValue; @@ -56,19 +57,14 @@ import org.apache.cassandra.io.sstable.Component; import org.apache.cassandra.io.sstable.Descriptor; import org.apache.cassandra.io.sstable.IndexSummary; import org.apache.cassandra.io.sstable.format.SSTableReader; -import org.apache.cassandra.io.sstable.format.big.BigTableReader; -import org.apache.cassandra.io.sstable.metadata.MetadataCollector; -import org.apache.cassandra.io.sstable.metadata.MetadataType; -import org.apache.cassandra.io.sstable.metadata.StatsMetadata; -import org.apache.cassandra.io.util.*; -import org.apache.cassandra.locator.SimpleStrategy; import org.apache.cassandra.service.StorageService; import org.apache.cassandra.utils.AlwaysPresentFilter; import org.apache.cassandra.utils.ByteBufferUtil; import org.apache.cassandra.utils.CounterId; -import org.apache.hadoop.fs.FileUtil; +import org.apache.cassandra.utils.FBUtilities; import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertNotNull; import static org.junit.Assert.assertTrue; public class Util @@ -90,60 +86,26 @@ public class Util return StorageService.getPartitioner().decorateKey(key); } - public static RowPosition rp(String key) + public static PartitionPosition rp(String key) { return rp(key, StorageService.getPartitioner()); } - public static RowPosition rp(String key, IPartitioner partitioner) + public static PartitionPosition rp(String key, IPartitioner partitioner) { - return RowPosition.ForKey.get(ByteBufferUtil.bytes(key), partitioner); + return PartitionPosition.ForKey.get(ByteBufferUtil.bytes(key), partitioner); } - public static CellName cellname(ByteBuffer... bbs) + public static Cell getRegularCell(CFMetaData metadata, Row row, String name) { - if (bbs.length == 1) - return CellNames.simpleDense(bbs[0]); - else - return CellNames.compositeDense(bbs); + ColumnDefinition column = metadata.getColumnDefinition(ByteBufferUtil.bytes(name)); + assert column != null; + return row.getCell(column); } - public static CellName cellname(String... strs) + public static ClusteringPrefix clustering(ClusteringComparator comparator, Object... o) { - ByteBuffer[] bbs = new ByteBuffer[strs.length]; - for (int i = 0; i < strs.length; i++) - bbs[i] = ByteBufferUtil.bytes(strs[i]); - return cellname(bbs); - } - - public static CellName cellname(int i) - { - return CellNames.simpleDense(ByteBufferUtil.bytes(i)); - } - - public static CellName cellname(long l) - { - return CellNames.simpleDense(ByteBufferUtil.bytes(l)); - } - - public static Cell column(String name, String value, long timestamp) - { - return new BufferCell(cellname(name), ByteBufferUtil.bytes(value), timestamp); - } - - public static Cell column(String name, long value, long timestamp) - { - return new BufferCell(cellname(name), ByteBufferUtil.bytes(value), timestamp); - } - - public static Cell column(String clusterKey, String name, long value, long timestamp) - { - return new BufferCell(cellname(clusterKey, name), ByteBufferUtil.bytes(value), timestamp); - } - - public static Cell expiringColumn(String name, String value, long timestamp, int ttl) - { - return new BufferExpiringCell(cellname(name), ByteBufferUtil.bytes(value), timestamp, ttl); + return comparator.make(o).clustering(); } public static Token token(String key) @@ -151,27 +113,28 @@ public class Util return StorageService.getPartitioner().getToken(ByteBufferUtil.bytes(key)); } - public static Range range(String left, String right) + public static Range range(String left, String right) { - return new Range(rp(left), rp(right)); + return new Range<>(rp(left), rp(right)); } - public static Range range(IPartitioner p, String left, String right) + public static Range range(IPartitioner p, String left, String right) { - return new Range(rp(left, p), rp(right, p)); + return new Range<>(rp(left, p), rp(right, p)); } - public static Bounds bounds(String left, String right) + //Test helper to make an iterator iterable once + public static Iterable once(final Iterator source) { - return new Bounds(rp(left), rp(right)); - } - - public static void addMutation(Mutation rm, String columnFamilyName, String superColumnName, long columnName, String value, long timestamp) - { - CellName cname = superColumnName == null - ? CellNames.simpleDense(getBytes(columnName)) - : CellNames.compositeDense(ByteBufferUtil.bytes(superColumnName), getBytes(columnName)); - rm.add(columnFamilyName, cname, ByteBufferUtil.bytes(value), timestamp); + return new Iterable() + { + private AtomicBoolean exhausted = new AtomicBoolean(); + public Iterator iterator() + { + Preconditions.checkState(!exhausted.getAndSet(true)); + return source; + } + }; } public static ByteBuffer getBytes(long v) @@ -192,39 +155,6 @@ public class Util return bb; } - public static ByteBuffer getBytes(short v) - { - byte[] bytes = new byte[2]; - ByteBuffer bb = ByteBuffer.wrap(bytes); - bb.putShort(v); - bb.rewind(); - return bb; - } - - public static ByteBuffer getBytes(byte v) - { - byte[] bytes = new byte[1]; - ByteBuffer bb = ByteBuffer.wrap(bytes); - bb.put(v); - bb.rewind(); - return bb; - } - - public static List getRangeSlice(ColumnFamilyStore cfs) - { - return getRangeSlice(cfs, null); - } - - public static List getRangeSlice(ColumnFamilyStore cfs, ByteBuffer superColumn) - { - IDiskAtomFilter filter = superColumn == null - ? new IdentityQueryFilter() - : new SliceQueryFilter(SuperColumns.startOf(superColumn), SuperColumns.endOf(superColumn), false, Integer.MAX_VALUE); - - Token min = StorageService.getPartitioner().getMinimumToken(); - return cfs.getRangeSlice(Bounds.makeRowBounds(min, min), null, filter, 10000); - } - /** * Writes out a bunch of mutations for a single column family. * @@ -245,23 +175,11 @@ public class Util return store; } - public static ColumnFamily getColumnFamily(Keyspace keyspace, DecoratedKey key, String cfName) - { - ColumnFamilyStore cfStore = keyspace.getColumnFamilyStore(cfName); - assert cfStore != null : "Table " + cfName + " has not been defined"; - return cfStore.getColumnFamily(QueryFilter.getIdentityFilter(key, cfName, System.currentTimeMillis())); - } - public static boolean equalsCounterId(CounterId n, ByteBuffer context, int offset) { return CounterId.wrap(context, context.position() + offset).equals(n); } - public static ColumnFamily cloneAndRemoveDeleted(ColumnFamily cf, int gcBefore) - { - return ColumnFamilyStore.removeDeleted(cf.cloneMe(), gcBefore); - } - /** * Creates initial set of nodes and tokens. Nodes are added to StorageService as 'normal' */ @@ -306,7 +224,7 @@ public class Util public static void compact(ColumnFamilyStore cfs, Collection sstables) { - int gcBefore = cfs.gcBefore(System.currentTimeMillis()); + int gcBefore = cfs.gcBefore(FBUtilities.nowInSeconds()); AbstractCompactionTask task = cfs.getCompactionStrategyManager().getUserDefinedTask(sstables, gcBefore); task.execute(null); } @@ -333,55 +251,235 @@ public class Util assert thrown : exception.getName() + " not received"; } - public static QueryFilter namesQueryFilter(ColumnFamilyStore cfs, DecoratedKey key) + public static AbstractReadCommandBuilder.SinglePartitionBuilder cmd(ColumnFamilyStore cfs, Object... partitionKey) { - SortedSet s = new TreeSet(cfs.getComparator()); - return QueryFilter.getNamesFilter(key, cfs.name, s, System.currentTimeMillis()); + return new AbstractReadCommandBuilder.SinglePartitionBuilder(cfs, makeKey(cfs.metadata, partitionKey)); } - public static QueryFilter namesQueryFilter(ColumnFamilyStore cfs, DecoratedKey key, String... names) + public static AbstractReadCommandBuilder.PartitionRangeBuilder cmd(ColumnFamilyStore cfs) { - SortedSet s = new TreeSet(cfs.getComparator()); - for (String str : names) - s.add(cellname(str)); - return QueryFilter.getNamesFilter(key, cfs.name, s, System.currentTimeMillis()); + return new AbstractReadCommandBuilder.PartitionRangeBuilder(cfs); } - public static QueryFilter namesQueryFilter(ColumnFamilyStore cfs, DecoratedKey key, CellName... names) + static DecoratedKey makeKey(CFMetaData metadata, Object... partitionKey) { - SortedSet s = new TreeSet(cfs.getComparator()); - for (CellName n : names) - s.add(n); - return QueryFilter.getNamesFilter(key, cfs.name, s, System.currentTimeMillis()); + if (partitionKey.length == 1 && partitionKey[0] instanceof DecoratedKey) + return (DecoratedKey)partitionKey[0]; + + ByteBuffer key = CFMetaData.serializePartitionKey(metadata.getKeyValidatorAsClusteringComparator().make(partitionKey)); + return StorageService.getPartitioner().decorateKey(key); } - public static NamesQueryFilter namesFilter(ColumnFamilyStore cfs, String... names) + public static void assertEmptyUnfiltered(ReadCommand command) { - SortedSet s = new TreeSet(cfs.getComparator()); - for (String str : names) - s.add(cellname(str)); - return new NamesQueryFilter(s); - } - - public static String string(ByteBuffer bb) - { - try + try (ReadOrderGroup orderGroup = command.startOrderGroup(); UnfilteredPartitionIterator iterator = command.executeLocally(orderGroup)) { - return ByteBufferUtil.string(bb); - } - catch (Exception e) - { - throw new RuntimeException(e); + if (iterator.hasNext()) + { + try (UnfilteredRowIterator partition = iterator.next()) + { + throw new AssertionError("Expected no results for query " + command.toCQLString() + " but got key " + command.metadata().getKeyValidator().getString(partition.partitionKey().getKey())); + } + } } } - public static RangeTombstone tombstone(String start, String finish, long timestamp, int localtime) + public static void assertEmpty(ReadCommand command) { - Composite startName = CellNames.simpleDense(ByteBufferUtil.bytes(start)); - Composite endName = CellNames.simpleDense(ByteBufferUtil.bytes(finish)); - return new RangeTombstone(startName, endName, timestamp , localtime); + try (ReadOrderGroup orderGroup = command.startOrderGroup(); PartitionIterator iterator = command.executeInternal(orderGroup)) + { + if (iterator.hasNext()) + { + try (RowIterator partition = iterator.next()) + { + throw new AssertionError("Expected no results for query " + command.toCQLString() + " but got key " + command.metadata().getKeyValidator().getString(partition.partitionKey().getKey())); + } + } + } } + public static List getAllUnfiltered(ReadCommand command) + { + List results = new ArrayList<>(); + try (ReadOrderGroup orderGroup = command.startOrderGroup(); UnfilteredPartitionIterator iterator = command.executeLocally(orderGroup)) + { + while (iterator.hasNext()) + { + try (UnfilteredRowIterator partition = iterator.next()) + { + results.add(ArrayBackedPartition.create(partition)); + } + } + } + return results; + } + + public static List getAll(ReadCommand command) + { + List results = new ArrayList<>(); + try (ReadOrderGroup orderGroup = command.startOrderGroup(); PartitionIterator iterator = command.executeInternal(orderGroup)) + { + while (iterator.hasNext()) + { + try (RowIterator partition = iterator.next()) + { + results.add(FilteredPartition.create(partition)); + } + } + } + return results; + } + + public static Row getOnlyRowUnfiltered(ReadCommand cmd) + { + try (ReadOrderGroup orderGroup = cmd.startOrderGroup(); UnfilteredPartitionIterator iterator = cmd.executeLocally(orderGroup)) + { + assert iterator.hasNext() : "Expecting one row in one partition but got nothing"; + try (UnfilteredRowIterator partition = iterator.next()) + { + assert !iterator.hasNext() : "Expecting a single partition but got more"; + + assert partition.hasNext() : "Expecting one row in one partition but got an empty partition"; + Row row = ((Row)partition.next()).takeAlias(); + assert !partition.hasNext() : "Expecting a single row but got more"; + return row; + } + } + } + + public static Row getOnlyRow(ReadCommand cmd) + { + try (ReadOrderGroup orderGroup = cmd.startOrderGroup(); PartitionIterator iterator = cmd.executeInternal(orderGroup)) + { + assert iterator.hasNext() : "Expecting one row in one partition but got nothing"; + try (RowIterator partition = iterator.next()) + { + assert !iterator.hasNext() : "Expecting a single partition but got more"; + assert partition.hasNext() : "Expecting one row in one partition but got an empty partition"; + Row row = partition.next().takeAlias(); + assert !partition.hasNext() : "Expecting a single row but got more"; + return row; + } + } + } + + public static ArrayBackedPartition getOnlyPartitionUnfiltered(ReadCommand cmd) + { + try (ReadOrderGroup orderGroup = cmd.startOrderGroup(); UnfilteredPartitionIterator iterator = cmd.executeLocally(orderGroup)) + { + assert iterator.hasNext() : "Expecting a single partition but got nothing"; + try (UnfilteredRowIterator partition = iterator.next()) + { + assert !iterator.hasNext() : "Expecting a single partition but got more"; + return ArrayBackedPartition.create(partition); + } + } + } + + public static FilteredPartition getOnlyPartition(ReadCommand cmd) + { + try (ReadOrderGroup orderGroup = cmd.startOrderGroup(); PartitionIterator iterator = cmd.executeInternal(orderGroup)) + { + assert iterator.hasNext() : "Expecting a single partition but got nothing"; + try (RowIterator partition = iterator.next()) + { + assert !iterator.hasNext() : "Expecting a single partition but got more"; + return FilteredPartition.create(partition); + } + } + } + + public static UnfilteredRowIterator apply(Mutation mutation) + { + mutation.apply(); + assert mutation.getPartitionUpdates().size() == 1; + return mutation.getPartitionUpdates().iterator().next().unfilteredIterator(); + } + + public static Cell cell(ColumnFamilyStore cfs, Row row, String columnName) + { + ColumnDefinition def = cfs.metadata.getColumnDefinition(ByteBufferUtil.bytes(columnName)); + assert def != null; + return row.getCell(def); + } + + public static Row row(Partition partition, Object... clustering) + { + return partition.getRow(partition.metadata().comparator.make(clustering)); + } + + public static void assertCellValue(Object value, ColumnFamilyStore cfs, Row row, String columnName) + { + Cell cell = cell(cfs, row, columnName); + assert cell != null : "Row " + row.toString(cfs.metadata) + " has no cell for " + columnName; + assertEquals(value, cell.column().type.compose(cell.value())); + } + + public static void consume(UnfilteredRowIterator iter) + { + try (UnfilteredRowIterator iterator = iter) + { + while (iter.hasNext()) + iter.next(); + } + } + + public static int size(PartitionIterator iter) + { + int size = 0; + while (iter.hasNext()) + { + ++size; + iter.next().close(); + } + return size; + } + + public static CBuilder getCBuilderForCFM(CFMetaData cfm) + { + List clusteringColumns = cfm.clusteringColumns(); + List> types = new ArrayList<>(clusteringColumns.size()); + for (ColumnDefinition def : clusteringColumns) + types.add(def.type); + return CBuilder.create(new ClusteringComparator(types)); + } + + // moved & refactored from KeyspaceTest in < 3.0 + public static void assertColumns(Row row, String... expectedColumnNames) + { + Iterator cells = row == null ? Iterators.emptyIterator() : row.iterator(); + String[] actual = Iterators.toArray(Iterators.transform(cells, new Function() + { + public String apply(Cell cell) + { + return cell.column().name.toString(); + } + }), String.class); + + assert Arrays.equals(actual, expectedColumnNames) + : String.format("Columns [%s])] is not expected [%s]", + ((row == null) ? "" : row.columns().toString()), + StringUtils.join(expectedColumnNames, ",")); + } + + public static void assertColumn(CFMetaData cfm, Row row, String name, String value, long timestamp) + { + Cell cell = row.getCell(cfm.getColumnDefinition(new ColumnIdentifier(name, true))); + assertColumn(cell, value, timestamp); + } + + public static void assertColumn(Cell cell, String value, long timestamp) + { + assertNotNull(cell); + assertEquals(0, ByteBufferUtil.compareUnsigned(cell.value(), ByteBufferUtil.bytes(value))); + assertEquals(timestamp, cell.livenessInfo().timestamp()); + } + + public static void assertClustering(CFMetaData cfm, Row row, Object... clusteringValue) + { + assertEquals(row.clustering().size(), clusteringValue.length); + assertEquals(0, cfm.comparator.compare(row.clustering(), cfm.comparator.make(clusteringValue))); + } public static void spinAssertEquals(Object expected, Supplier s, int timeoutInSeconds) { diff --git a/test/unit/org/apache/cassandra/cache/AutoSavingCacheTest.java b/test/unit/org/apache/cassandra/cache/AutoSavingCacheTest.java index c1869b980d..e7a1706e9b 100644 --- a/test/unit/org/apache/cassandra/cache/AutoSavingCacheTest.java +++ b/test/unit/org/apache/cassandra/cache/AutoSavingCacheTest.java @@ -17,6 +17,11 @@ */ package org.apache.cassandra.cache; +import org.apache.cassandra.config.CFMetaData; +import org.apache.cassandra.config.ColumnDefinition; +import org.apache.cassandra.cql3.ColumnIdentifier; +import org.apache.cassandra.db.*; +import org.apache.cassandra.db.marshal.AsciiType; import org.apache.cassandra.io.sstable.format.SSTableReader; import org.junit.Assert; import org.junit.BeforeClass; @@ -25,10 +30,6 @@ import org.junit.Test; import org.apache.cassandra.SchemaLoader; import org.apache.cassandra.Util; import org.apache.cassandra.config.KSMetaData; -import org.apache.cassandra.db.ColumnFamilyStore; -import org.apache.cassandra.db.Keyspace; -import org.apache.cassandra.db.RowIndexEntry; -import org.apache.cassandra.db.Mutation; import org.apache.cassandra.exceptions.ConfigurationException; import org.apache.cassandra.locator.SimpleStrategy; import org.apache.cassandra.service.CacheService; @@ -44,9 +45,12 @@ public class AutoSavingCacheTest { SchemaLoader.prepareServer(); SchemaLoader.createKeyspace(KEYSPACE1, - SimpleStrategy.class, - KSMetaData.optsWithRF(1), - SchemaLoader.standardCFMD(KEYSPACE1, CF_STANDARD1)); + SimpleStrategy.class, + KSMetaData.optsWithRF(1), + CFMetaData.Builder.create(KEYSPACE1, CF_STANDARD1) + .addPartitionKey("pKey", AsciiType.instance) + .addRegularColumn("col1", AsciiType.instance) + .build()); } @Test @@ -55,9 +59,10 @@ public class AutoSavingCacheTest ColumnFamilyStore cfs = Keyspace.open(KEYSPACE1).getColumnFamilyStore(CF_STANDARD1); for (int i = 0; i < 2; i++) { - Mutation rm = new Mutation(KEYSPACE1, ByteBufferUtil.bytes("key1")); - rm.add(CF_STANDARD1, Util.cellname("c1"), ByteBufferUtil.bytes(i), 0); - rm.applyUnsafe(); + ColumnDefinition colDef = new ColumnDefinition(cfs.metadata, ByteBufferUtil.bytes("col1"), AsciiType.instance, 0, ColumnDefinition.Kind.REGULAR); + RowUpdateBuilder rowBuilder = new RowUpdateBuilder(cfs.metadata, System.currentTimeMillis(), "key1"); + rowBuilder.add(colDef, "val1"); + rowBuilder.build().apply(); cfs.forceBlockingFlush(); } diff --git a/test/unit/org/apache/cassandra/cache/CacheProviderTest.java b/test/unit/org/apache/cassandra/cache/CacheProviderTest.java index fe266164fc..d92d427493 100644 --- a/test/unit/org/apache/cassandra/cache/CacheProviderTest.java +++ b/test/unit/org/apache/cassandra/cache/CacheProviderTest.java @@ -22,24 +22,27 @@ package org.apache.cassandra.cache; import java.nio.ByteBuffer; -import java.util.ArrayList; -import java.util.List; -import java.util.UUID; +import java.security.MessageDigest; +import java.security.NoSuchAlgorithmException; +import java.util.*; +import org.apache.cassandra.Util; import org.junit.BeforeClass; import org.junit.Test; - -import org.apache.cassandra.SchemaLoader; -import org.apache.cassandra.config.KSMetaData; -import org.apache.cassandra.db.ArrayBackedSortedColumns; -import org.apache.cassandra.db.ColumnFamily; -import org.apache.cassandra.exceptions.ConfigurationException; -import org.apache.cassandra.locator.SimpleStrategy; +import static org.junit.Assert.*; import com.googlecode.concurrentlinkedhashmap.Weighers; -import static org.apache.cassandra.Util.column; -import static org.junit.Assert.*; +import org.apache.cassandra.config.CFMetaData; +import org.apache.cassandra.config.KSMetaData; +import org.apache.cassandra.db.*; +import org.apache.cassandra.db.rows.*; +import org.apache.cassandra.db.marshal.AsciiType; +import org.apache.cassandra.db.partitions.*; +import org.apache.cassandra.exceptions.ConfigurationException; +import org.apache.cassandra.locator.SimpleStrategy; +import org.apache.cassandra.SchemaLoader; +import org.apache.cassandra.utils.FBUtilities; public class CacheProviderTest { @@ -52,59 +55,86 @@ public class CacheProviderTest private static final String KEYSPACE1 = "CacheProviderTest1"; private static final String CF_STANDARD1 = "Standard1"; + private static CFMetaData cfm; + @BeforeClass public static void defineSchema() throws ConfigurationException { SchemaLoader.prepareServer(); + + cfm = CFMetaData.Builder.create(KEYSPACE1, CF_STANDARD1) + .addPartitionKey("pKey", AsciiType.instance) + .addRegularColumn("col1", AsciiType.instance) + .build(); SchemaLoader.createKeyspace(KEYSPACE1, SimpleStrategy.class, KSMetaData.optsWithRF(1), - SchemaLoader.standardCFMD(KEYSPACE1, CF_STANDARD1)); + cfm); } - private void simpleCase(ColumnFamily cf, ICache cache) + private ArrayBackedCachedPartition createPartition() { - cache.put(key1, cf); + PartitionUpdate update = new RowUpdateBuilder(cfm, System.currentTimeMillis(), "key1") + .add("col1", "val1") + .buildUpdate(); + + return ArrayBackedCachedPartition.create(update.unfilteredIterator(), FBUtilities.nowInSeconds()); + } + + private void simpleCase(ArrayBackedCachedPartition partition, ICache cache) + { + cache.put(key1, partition); assertNotNull(cache.get(key1)); - assertDigests(cache.get(key1), cf); - cache.put(key2, cf); - cache.put(key3, cf); - cache.put(key4, cf); - cache.put(key5, cf); + assertDigests(cache.get(key1), partition); + cache.put(key2, partition); + cache.put(key3, partition); + cache.put(key4, partition); + cache.put(key5, partition); assertEquals(CAPACITY, cache.size()); } - private void assertDigests(IRowCacheEntry one, ColumnFamily two) + private void assertDigests(IRowCacheEntry one, ArrayBackedCachedPartition two) { - // CF does not implement .equals - assertTrue(one instanceof ColumnFamily); - assertEquals(ColumnFamily.digest((ColumnFamily)one), ColumnFamily.digest(two)); + assertTrue(one instanceof ArrayBackedCachedPartition); + try + { + MessageDigest d1 = MessageDigest.getInstance("MD5"); + MessageDigest d2 = MessageDigest.getInstance("MD5"); + UnfilteredRowIterators.digest(((ArrayBackedCachedPartition) one).unfilteredIterator(), d1); + UnfilteredRowIterators.digest(((ArrayBackedCachedPartition) two).unfilteredIterator(), d2); + assertTrue(MessageDigest.isEqual(d1.digest(), d2.digest())); + } + catch (NoSuchAlgorithmException e) + { + throw new RuntimeException(e); + } } - // TODO this isn't terribly useful - private void concurrentCase(final ColumnFamily cf, final ICache cache) throws InterruptedException + private void concurrentCase(final ArrayBackedCachedPartition partition, final ICache cache) throws InterruptedException { - Runnable runable = new Runnable() + final long startTime = System.currentTimeMillis() + 500; + Runnable runnable = new Runnable() { public void run() { - for (int j = 0; j < 10; j++) + while (System.currentTimeMillis() < startTime) {} + for (int j = 0; j < 1000; j++) { - cache.put(key1, cf); - cache.put(key2, cf); - cache.put(key3, cf); - cache.put(key4, cf); - cache.put(key5, cf); + cache.put(key1, partition); + cache.put(key2, partition); + cache.put(key3, partition); + cache.put(key4, partition); + cache.put(key5, partition); } } }; - List threads = new ArrayList(100); + List threads = new ArrayList<>(100); for (int i = 0; i < 100; i++) { - Thread thread = new Thread(runable); + Thread thread = new Thread(runnable); threads.add(thread); thread.start(); } @@ -112,28 +142,21 @@ public class CacheProviderTest thread.join(); } - private ColumnFamily createCF() - { - ColumnFamily cf = ArrayBackedSortedColumns.factory.create(KEYSPACE1, CF_STANDARD1); - cf.addColumn(column("vijay", "great", 1)); - cf.addColumn(column("awesome", "vijay", 1)); - return cf; - } - @Test public void testSerializingCache() throws InterruptedException { ICache cache = SerializingCache.create(CAPACITY, Weighers.singleton(), new SerializingCacheProvider.RowCacheSerializer()); - ColumnFamily cf = createCF(); - simpleCase(cf, cache); - concurrentCase(cf, cache); + ArrayBackedCachedPartition partition = createPartition(); + simpleCase(partition, cache); + concurrentCase(partition, cache); } - + @Test public void testKeys() { UUID cfId = UUID.randomUUID(); + byte[] b1 = {1, 2, 3, 4}; RowCacheKey key1 = new RowCacheKey(cfId, ByteBuffer.wrap(b1)); byte[] b2 = {1, 2, 3, 4}; diff --git a/test/unit/org/apache/cassandra/config/CFMetaDataTest.java b/test/unit/org/apache/cassandra/config/CFMetaDataTest.java new file mode 100644 index 0000000000..4a69a7c090 --- /dev/null +++ b/test/unit/org/apache/cassandra/config/CFMetaDataTest.java @@ -0,0 +1,155 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.cassandra.config; + +import java.util.ArrayList; +import java.util.List; +import java.util.HashMap; +import java.util.HashSet; + +import org.apache.cassandra.SchemaLoader; +import org.apache.cassandra.db.*; +import org.apache.cassandra.db.marshal.AsciiType; +import org.apache.cassandra.db.marshal.UTF8Type; +import org.apache.cassandra.db.rows.UnfilteredRowIterators; +import org.apache.cassandra.db.partitions.PartitionUpdate; +import org.apache.cassandra.exceptions.ConfigurationException; +import org.apache.cassandra.io.compress.*; +import org.apache.cassandra.locator.SimpleStrategy; +import org.apache.cassandra.schema.LegacySchemaTables; +import org.apache.cassandra.service.StorageService; +import org.apache.cassandra.thrift.CfDef; +import org.apache.cassandra.thrift.ColumnDef; +import org.apache.cassandra.thrift.IndexType; +import org.apache.cassandra.thrift.ThriftConversion; +import org.apache.cassandra.utils.ByteBufferUtil; +import org.apache.cassandra.utils.FBUtilities; + +import org.junit.BeforeClass; +import org.junit.Test; + +import static org.junit.Assert.assertEquals; + +public class CFMetaDataTest +{ + private static final String KEYSPACE1 = "CFMetaDataTest1"; + private static final String CF_STANDARD1 = "Standard1"; + + private static List columnDefs = new ArrayList(); + + static + { + columnDefs.add(new ColumnDef(ByteBufferUtil.bytes("col1"), AsciiType.class.getCanonicalName()) + .setIndex_name("col1Index") + .setIndex_type(IndexType.KEYS)); + + columnDefs.add(new ColumnDef(ByteBufferUtil.bytes("col2"), UTF8Type.class.getCanonicalName()) + .setIndex_name("col2Index") + .setIndex_type(IndexType.KEYS)); + } + + @BeforeClass + public static void defineSchema() throws ConfigurationException + { + SchemaLoader.prepareServer(); + SchemaLoader.createKeyspace(KEYSPACE1, + SimpleStrategy.class, + KSMetaData.optsWithRF(1), + SchemaLoader.standardCFMD(KEYSPACE1, CF_STANDARD1)); + } + + @Test + public void testThriftConversion() throws Exception + { + CfDef cfDef = new CfDef().setDefault_validation_class(AsciiType.class.getCanonicalName()) + .setComment("Test comment") + .setColumn_metadata(columnDefs) + .setKeyspace(KEYSPACE1) + .setName(CF_STANDARD1); + + // convert Thrift to CFMetaData + CFMetaData cfMetaData = ThriftConversion.fromThrift(cfDef); + + CfDef thriftCfDef = new CfDef(); + thriftCfDef.keyspace = KEYSPACE1; + thriftCfDef.name = CF_STANDARD1; + thriftCfDef.default_validation_class = cfDef.default_validation_class; + thriftCfDef.comment = cfDef.comment; + thriftCfDef.column_metadata = new ArrayList<>(); + for (ColumnDef columnDef : columnDefs) + { + ColumnDef c = new ColumnDef(); + c.name = ByteBufferUtil.clone(columnDef.name); + c.validation_class = columnDef.getValidation_class(); + c.index_name = columnDef.getIndex_name(); + c.index_type = IndexType.KEYS; + thriftCfDef.column_metadata.add(c); + } + + CfDef converted = ThriftConversion.toThrift(cfMetaData); + + assertEquals(thriftCfDef.keyspace, converted.keyspace); + assertEquals(thriftCfDef.name, converted.name); + assertEquals(thriftCfDef.default_validation_class, converted.default_validation_class); + assertEquals(thriftCfDef.comment, converted.comment); + assertEquals(new HashSet<>(thriftCfDef.column_metadata), new HashSet<>(converted.column_metadata)); + } + + @Test + public void testConversionsInverses() throws Exception + { + for (String keyspaceName : Schema.instance.getNonSystemKeyspaces()) + { + for (ColumnFamilyStore cfs : Keyspace.open(keyspaceName).getColumnFamilyStores()) + { + CFMetaData cfm = cfs.metadata; + if (!cfm.isThriftCompatible()) + continue; + + checkInverses(cfm); + + // Testing with compression to catch #3558 + CFMetaData withCompression = cfm.copy(); + withCompression.compressionParameters(new CompressionParameters(SnappyCompressor.instance, 32768, new HashMap())); + checkInverses(withCompression); + } + } + } + + private void checkInverses(CFMetaData cfm) throws Exception + { + DecoratedKey k = StorageService.getPartitioner().decorateKey(ByteBufferUtil.bytes(cfm.ksName)); + KSMetaData keyspace = Schema.instance.getKSMetaData(cfm.ksName); + + // Test thrift conversion + CFMetaData before = cfm; + CFMetaData after = ThriftConversion.fromThriftForUpdate(ThriftConversion.toThrift(before), before); + assert before.equals(after) : String.format("%n%s%n!=%n%s", before, after); + + // Test schema conversion + Mutation rm = LegacySchemaTables.makeCreateTableMutation(keyspace, cfm, FBUtilities.timestampMicros()); + PartitionUpdate cfU = rm.getPartitionUpdate(Schema.instance.getId(SystemKeyspace.NAME, LegacySchemaTables.COLUMNFAMILIES)); + PartitionUpdate cdU = rm.getPartitionUpdate(Schema.instance.getId(SystemKeyspace.NAME, LegacySchemaTables.COLUMNS)); + CFMetaData newCfm = LegacySchemaTables.createTableFromTablePartitionAndColumnsPartition( + UnfilteredRowIterators.filter(cfU.unfilteredIterator(), FBUtilities.nowInSeconds()), + UnfilteredRowIterators.filter(cdU.unfilteredIterator(), FBUtilities.nowInSeconds()) + ); + assert cfm.equals(newCfm) : String.format("%n%s%n!=%n%s", cfm, newCfm); + } +} diff --git a/test/unit/org/apache/cassandra/config/ColumnDefinitionTest.java b/test/unit/org/apache/cassandra/config/ColumnDefinitionTest.java index 2bee0c37d2..0e5e19267a 100644 --- a/test/unit/org/apache/cassandra/config/ColumnDefinitionTest.java +++ b/test/unit/org/apache/cassandra/config/ColumnDefinitionTest.java @@ -23,7 +23,6 @@ package org.apache.cassandra.config; import org.junit.Assert; import org.junit.Test; -import org.apache.cassandra.config.CFMetaData; import org.apache.cassandra.db.marshal.*; import org.apache.cassandra.thrift.ThriftConversion; import org.apache.cassandra.utils.ByteBufferUtil; @@ -33,12 +32,16 @@ public class ColumnDefinitionTest @Test public void testSerializeDeserialize() throws Exception { - CFMetaData cfm = CFMetaData.denseCFMetaData("ks", "cf", UTF8Type.instance); + CFMetaData cfm = CFMetaData.Builder.create("ks", "cf", true, false, false) + .addPartitionKey("pkey", AsciiType.instance) + .addClusteringColumn("name", AsciiType.instance) + .addRegularColumn("val", AsciiType.instance) + .build(); - ColumnDefinition cd0 = ColumnDefinition.regularDef(cfm, ByteBufferUtil.bytes("TestColumnDefinitionName0"), BytesType.instance, null) + ColumnDefinition cd0 = ColumnDefinition.staticDef(cfm, ByteBufferUtil.bytes("TestColumnDefinitionName0"), BytesType.instance, null) .setIndex("random index name 0", IndexType.KEYS, null); - ColumnDefinition cd1 = ColumnDefinition.regularDef(cfm, ByteBufferUtil.bytes("TestColumnDefinition1"), LongType.instance, null); + ColumnDefinition cd1 = ColumnDefinition.staticDef(cfm, ByteBufferUtil.bytes("TestColumnDefinition1"), LongType.instance, null); testSerializeDeserialize(cfm, cd0); testSerializeDeserialize(cfm, cd1); @@ -46,7 +49,7 @@ public class ColumnDefinitionTest protected void testSerializeDeserialize(CFMetaData cfm, ColumnDefinition cd) throws Exception { - ColumnDefinition newCd = ThriftConversion.fromThrift(cfm.ksName, cfm.cfName, cfm.comparator.asAbstractType(), null, ThriftConversion.toThrift(cd)); + ColumnDefinition newCd = ThriftConversion.fromThrift(cfm.ksName, cfm.cfName, cfm.comparator.subtype(0), null, ThriftConversion.toThrift(cd)); Assert.assertNotSame(cd, newCd); Assert.assertEquals(cd.hashCode(), newCd.hashCode()); Assert.assertEquals(cd, newCd); diff --git a/test/unit/org/apache/cassandra/config/LegacySchemaTablesTest.java b/test/unit/org/apache/cassandra/config/LegacySchemaTablesTest.java index 3642e7a504..b7a2a379e1 100644 --- a/test/unit/org/apache/cassandra/config/LegacySchemaTablesTest.java +++ b/test/unit/org/apache/cassandra/config/LegacySchemaTablesTest.java @@ -25,6 +25,8 @@ import java.util.HashSet; import org.apache.cassandra.SchemaLoader; import org.apache.cassandra.db.*; +import org.apache.cassandra.db.rows.UnfilteredRowIterators; +import org.apache.cassandra.db.partitions.PartitionUpdate; import org.apache.cassandra.db.marshal.AsciiType; import org.apache.cassandra.db.marshal.UTF8Type; import org.apache.cassandra.exceptions.ConfigurationException; @@ -142,9 +144,10 @@ public class LegacySchemaTablesTest // Test schema conversion Mutation rm = LegacySchemaTables.makeCreateTableMutation(keyspace, cfm, FBUtilities.timestampMicros()); - ColumnFamily serializedCf = rm.getColumnFamily(Schema.instance.getId(SystemKeyspace.NAME, LegacySchemaTables.COLUMNFAMILIES)); - ColumnFamily serializedCD = rm.getColumnFamily(Schema.instance.getId(SystemKeyspace.NAME, LegacySchemaTables.COLUMNS)); - CFMetaData newCfm = LegacySchemaTables.createTableFromTablePartitionAndColumnsPartition(new Row(k, serializedCf), new Row(k, serializedCD)); + PartitionUpdate serializedCf = rm.getPartitionUpdate(Schema.instance.getId(SystemKeyspace.NAME, LegacySchemaTables.COLUMNFAMILIES)); + PartitionUpdate serializedCD = rm.getPartitionUpdate(Schema.instance.getId(SystemKeyspace.NAME, LegacySchemaTables.COLUMNS)); + CFMetaData newCfm = LegacySchemaTables.createTableFromTablePartitionAndColumnsPartition(UnfilteredRowIterators.filter(serializedCf.unfilteredIterator(), FBUtilities.nowInSeconds()), + UnfilteredRowIterators.filter(serializedCD.unfilteredIterator(), FBUtilities.nowInSeconds())); assert cfm.equals(newCfm) : String.format("%n%s%n!=%n%s", cfm, newCfm); } } diff --git a/test/unit/org/apache/cassandra/cql3/CQLTester.java b/test/unit/org/apache/cassandra/cql3/CQLTester.java index 4b4631e906..2e8f3b36d8 100644 --- a/test/unit/org/apache/cassandra/cql3/CQLTester.java +++ b/test/unit/org/apache/cassandra/cql3/CQLTester.java @@ -18,6 +18,7 @@ package org.apache.cassandra.cql3; import java.io.File; +import java.io.IOException; import java.math.BigDecimal; import java.math.BigInteger; import java.net.InetAddress; @@ -29,20 +30,14 @@ import java.util.concurrent.ExecutionException; import java.util.concurrent.TimeUnit; import java.util.concurrent.atomic.AtomicInteger; +import com.datastax.driver.core.*; +import com.datastax.driver.core.ResultSet; import com.google.common.base.Objects; import com.google.common.collect.ImmutableSet; import org.junit.*; import org.slf4j.Logger; import org.slf4j.LoggerFactory; -import static junit.framework.Assert.assertNotNull; -import com.datastax.driver.core.Cluster; -import com.datastax.driver.core.ColumnDefinitions; -import com.datastax.driver.core.DataType; -import com.datastax.driver.core.ProtocolVersion; -import com.datastax.driver.core.ResultSet; -import com.datastax.driver.core.Row; -import com.datastax.driver.core.Session; import org.apache.cassandra.SchemaLoader; import org.apache.cassandra.concurrent.ScheduledExecutors; import org.apache.cassandra.config.CFMetaData; @@ -50,15 +45,13 @@ import org.apache.cassandra.config.DatabaseDescriptor; import org.apache.cassandra.config.Schema; import org.apache.cassandra.cql3.functions.FunctionName; import org.apache.cassandra.cql3.statements.ParsedStatement; -import org.apache.cassandra.db.Directories; -import org.apache.cassandra.db.Keyspace; -import org.apache.cassandra.db.SystemKeyspace; +import org.apache.cassandra.db.*; +import org.apache.cassandra.db.commitlog.CommitLog; import org.apache.cassandra.db.marshal.*; import org.apache.cassandra.db.marshal.TupleType; -import org.apache.cassandra.exceptions.CassandraException; -import org.apache.cassandra.exceptions.ConfigurationException; -import org.apache.cassandra.exceptions.SyntaxException; import org.apache.cassandra.dht.Murmur3Partitioner; +import org.apache.cassandra.exceptions.*; +import org.apache.cassandra.gms.Gossiper; import org.apache.cassandra.io.util.FileUtils; import org.apache.cassandra.serializers.TypeSerializer; import org.apache.cassandra.service.ClientState; @@ -69,6 +62,8 @@ import org.apache.cassandra.transport.Server; import org.apache.cassandra.transport.messages.ResultMessage; import org.apache.cassandra.utils.ByteBufferUtil; +import static junit.framework.Assert.assertNotNull; + /** * Base class for CQL tests. */ @@ -88,8 +83,11 @@ public abstract class CQLTester private static final Cluster[] cluster; private static final Session[] session; + private static boolean isServerPrepared = false; + public static int maxProtocolVersion; - static { + static + { int version; for (version = 1; version <= Server.CURRENT_VERSION; ) { @@ -108,7 +106,7 @@ public abstract class CQLTester session = new Session[maxProtocolVersion]; // Once per-JVM is enough - SchemaLoader.prepareServer(); + prepareServer(true); nativeAddr = InetAddress.getLoopbackAddress(); @@ -137,13 +135,90 @@ public abstract class CQLTester // is not expected to be the same without preparation) private boolean usePrepared = USE_PREPARED_VALUES; + public static void prepareServer(boolean checkInit) + { + if (checkInit && isServerPrepared) + return; + + // Cleanup first + try + { + cleanupAndLeaveDirs(); + } + catch (IOException e) + { + logger.error("Failed to cleanup and recreate directories."); + throw new RuntimeException(e); + } + + Thread.setDefaultUncaughtExceptionHandler(new Thread.UncaughtExceptionHandler() + { + public void uncaughtException(Thread t, Throwable e) + { + logger.error("Fatal exception in thread " + t, e); + } + }); + + Keyspace.setInitialized(); + isServerPrepared = true; + } + + public static void cleanupAndLeaveDirs() throws IOException + { + // We need to stop and unmap all CLS instances prior to cleanup() or we'll get failures on Windows. + CommitLog.instance.stopUnsafe(true); + mkdirs(); + cleanup(); + mkdirs(); + CommitLog.instance.startUnsafe(); + } + + public static void cleanup() + { + // clean up commitlog + String[] directoryNames = { DatabaseDescriptor.getCommitLogLocation(), }; + for (String dirName : directoryNames) + { + File dir = new File(dirName); + if (!dir.exists()) + throw new RuntimeException("No such directory: " + dir.getAbsolutePath()); + FileUtils.deleteRecursive(dir); + } + + cleanupSavedCaches(); + + // clean up data directory which are stored as data directory/keyspace/data files + for (String dirName : DatabaseDescriptor.getAllDataFileLocations()) + { + File dir = new File(dirName); + if (!dir.exists()) + throw new RuntimeException("No such directory: " + dir.getAbsolutePath()); + FileUtils.deleteRecursive(dir); + } + } + + public static void mkdirs() + { + DatabaseDescriptor.createAllDirectories(); + } + + public static void cleanupSavedCaches() + { + File cachesDir = new File(DatabaseDescriptor.getSavedCachesLocation()); + + if (!cachesDir.exists() || !cachesDir.isDirectory()) + return; + + FileUtils.delete(cachesDir.listFiles()); + } + @BeforeClass public static void setUpClass() { if (ROW_CACHE_SIZE_IN_MB > 0) DatabaseDescriptor.setRowCacheSizeInMB(ROW_CACHE_SIZE_IN_MB); - DatabaseDescriptor.setPartitioner(Murmur3Partitioner.instance); + StorageService.instance.setPartitionerUnsafe(Murmur3Partitioner.instance); } @AfterClass @@ -271,13 +346,21 @@ public abstract class CQLTester return list.isEmpty() ? Collections.emptyList() : new ArrayList<>(list); } + public ColumnFamilyStore getCurrentColumnFamilyStore() + { + String currentTable = currentTable(); + return currentTable == null + ? null + : Keyspace.open(KEYSPACE).getColumnFamilyStore(currentTable); + } + public void flush() { try { - String currentTable = currentTable(); - if (currentTable != null) - Keyspace.open(KEYSPACE).getColumnFamilyStore(currentTable).forceFlush().get(); + ColumnFamilyStore store = getCurrentColumnFamilyStore(); + if (store != null) + store.forceFlush().get(); } catch (InterruptedException | ExecutionException e) { @@ -570,17 +653,22 @@ public abstract class CQLTester UntypedResultSet rs; if (usePrepared) { - logger.info("Executing: {} with values {}", query, formatAllValues(values)); + if (logger.isDebugEnabled()) + logger.debug("Executing: {} with values {}", query, formatAllValues(values)); rs = QueryProcessor.executeOnceInternal(query, transformValues(values)); } else { query = replaceValues(query, values); - logger.info("Executing: {}", query); + if (logger.isDebugEnabled()) + logger.debug("Executing: {}", query); rs = QueryProcessor.executeOnceInternal(query); } if (rs != null) - logger.info("Got {} rows", rs.size()); + { + if (logger.isDebugEnabled()) + logger.debug("Got {} rows", rs.size()); + } return rs; } @@ -642,7 +730,7 @@ public abstract class CQLTester rows.length>i ? "less" : "more", rows.length, i, protocolVersion), i == rows.length); } - protected void assertRows(UntypedResultSet result, Object[]... rows) + public static void assertRows(UntypedResultSet result, Object[]... rows) { if (result == null) { @@ -690,7 +778,7 @@ public abstract class CQLTester iter.next(); i++; } - Assert.fail(String.format("Got less rows than expected. Expected %d but got %d.", rows.length, i)); + Assert.fail(String.format("Got more rows than expected. Expected %d but got %d.", rows.length, i)); } Assert.assertTrue(String.format("Got %s rows than expected. Expected %d but got %d", rows.length>i ? "less" : "more", rows.length, i), i == rows.length); @@ -778,7 +866,7 @@ public abstract class CQLTester assertRows(execute("SELECT * FROM %s"), rows); } - protected Object[] row(Object... expected) + public static Object[] row(Object... expected) { return expected; } diff --git a/test/unit/org/apache/cassandra/cql3/ColumnConditionTest.java b/test/unit/org/apache/cassandra/cql3/ColumnConditionTest.java index c8b3a2fc41..9a768de4b0 100644 --- a/test/unit/org/apache/cassandra/cql3/ColumnConditionTest.java +++ b/test/unit/org/apache/cassandra/cql3/ColumnConditionTest.java @@ -17,27 +17,28 @@ */ package org.apache.cassandra.cql3; +import java.nio.ByteBuffer; +import java.util.*; + +import org.junit.Test; + import org.apache.cassandra.config.CFMetaData; import org.apache.cassandra.config.ColumnDefinition; -import org.apache.cassandra.db.BufferCell; -import org.apache.cassandra.db.Cell; -import org.apache.cassandra.db.composites.*; +import org.apache.cassandra.db.LivenessInfo; +import org.apache.cassandra.db.rows.AbstractCell; +import org.apache.cassandra.db.rows.Cell; +import org.apache.cassandra.db.rows.CellPath; import org.apache.cassandra.db.marshal.*; import org.apache.cassandra.exceptions.InvalidRequestException; import org.apache.cassandra.serializers.Int32Serializer; import org.apache.cassandra.utils.ByteBufferUtil; -import org.junit.Test; -import java.nio.ByteBuffer; -import java.util.*; - -import static org.apache.cassandra.utils.ByteBufferUtil.UNSET_BYTE_BUFFER; -import static org.junit.Assert.assertTrue; -import static org.junit.Assert.assertFalse; -import static org.junit.Assert.fail; +import static org.junit.Assert.*; public class ColumnConditionTest { + public static ByteBuffer UNSET_BYTE_BUFFER = ByteBuffer.wrap(new byte[]{}); + public static final ByteBuffer ZERO = Int32Type.instance.fromString("0"); public static final ByteBuffer ONE = Int32Type.instance.fromString("1"); public static final ByteBuffer TWO = Int32Type.instance.fromString("2"); @@ -50,11 +51,10 @@ public class ColumnConditionTest Cell cell = null; if (columnValue != null) { - CompoundSparseCellNameType nameType = new CompoundSparseCellNameType(Collections.EMPTY_LIST); - ColumnDefinition definition = new ColumnDefinition("ks", "cf", new ColumnIdentifier("c", true), Int32Type.instance, null, null, null, null, null); - cell = new BufferCell(nameType.create(Composites.EMPTY, definition), columnValue); + ColumnDefinition definition = ColumnDefinition.regularDef("ks", "cf", "c", ListType.getInstance(Int32Type.instance, true), null); + cell = new TestCell(definition, null, columnValue, LivenessInfo.NONE); } - return bound.isSatisfiedByValue(conditionValue, cell, Int32Type.instance, bound.operator, 1234); + return bound.isSatisfiedByValue(conditionValue, cell, Int32Type.instance, bound.operator); } private static void assertThrowsIRE(ColumnCondition.Bound bound, ByteBuffer conditionValue, ByteBuffer columnValue) @@ -69,7 +69,7 @@ public class ColumnConditionTest @Test public void testSimpleBoundIsSatisfiedByValue() throws InvalidRequestException { - ColumnDefinition definition = new ColumnDefinition("ks", "cf", new ColumnIdentifier("c", true), Int32Type.instance, null, null, null, null, null); + ColumnDefinition definition = ColumnDefinition.regularDef("ks", "cf", "c", ListType.getInstance(Int32Type.instance, true), null); // EQ ColumnCondition condition = ColumnCondition.condition(definition, new Constants.Value(ONE), Operator.EQ); @@ -83,7 +83,6 @@ public class ColumnConditionTest assertTrue(isSatisfiedBy(bound, null, null)); assertFalse(isSatisfiedBy(bound, ONE, null)); assertFalse(isSatisfiedBy(bound, null, ONE)); - assertThrowsIRE(bound, UNSET_BYTE_BUFFER, ONE); // NEQ condition = ColumnCondition.condition(definition, new Constants.Value(ONE), Operator.NEQ); @@ -97,7 +96,6 @@ public class ColumnConditionTest assertFalse(isSatisfiedBy(bound, null, null)); assertTrue(isSatisfiedBy(bound, ONE, null)); assertTrue(isSatisfiedBy(bound, null, ONE)); - assertThrowsIRE(bound, UNSET_BYTE_BUFFER, ONE); // LT condition = ColumnCondition.condition(definition, new Constants.Value(ONE), Operator.LT); @@ -110,7 +108,6 @@ public class ColumnConditionTest assertFalse(isSatisfiedBy(bound, ByteBufferUtil.EMPTY_BYTE_BUFFER, ByteBufferUtil.EMPTY_BYTE_BUFFER)); assertThrowsIRE(bound, null, ONE); assertFalse(isSatisfiedBy(bound, ONE, null)); - assertThrowsIRE(bound, UNSET_BYTE_BUFFER, ONE); // LTE condition = ColumnCondition.condition(definition, new Constants.Value(ONE), Operator.LTE); @@ -123,7 +120,6 @@ public class ColumnConditionTest assertTrue(isSatisfiedBy(bound, ByteBufferUtil.EMPTY_BYTE_BUFFER, ByteBufferUtil.EMPTY_BYTE_BUFFER)); assertThrowsIRE(bound, null, ONE); assertFalse(isSatisfiedBy(bound, ONE, null)); - assertThrowsIRE(bound, UNSET_BYTE_BUFFER, ONE); // GT condition = ColumnCondition.condition(definition, new Constants.Value(ONE), Operator.GT); @@ -136,7 +132,6 @@ public class ColumnConditionTest assertFalse(isSatisfiedBy(bound, ByteBufferUtil.EMPTY_BYTE_BUFFER, ByteBufferUtil.EMPTY_BYTE_BUFFER)); assertThrowsIRE(bound, null, ONE); assertFalse(isSatisfiedBy(bound, ONE, null)); - assertThrowsIRE(bound, UNSET_BYTE_BUFFER, ONE); // GT condition = ColumnCondition.condition(definition, new Constants.Value(ONE), Operator.GTE); @@ -149,7 +144,6 @@ public class ColumnConditionTest assertTrue(isSatisfiedBy(bound, ByteBufferUtil.EMPTY_BYTE_BUFFER, ByteBufferUtil.EMPTY_BYTE_BUFFER)); assertThrowsIRE(bound, null, ONE); assertFalse(isSatisfiedBy(bound, ONE, null)); - assertThrowsIRE(bound, UNSET_BYTE_BUFFER, ONE); } private static List list(ByteBuffer... values) @@ -162,7 +156,7 @@ public class ColumnConditionTest CFMetaData cfm = CFMetaData.compile("create table foo(a int PRIMARY KEY, b int, c list)", "ks"); Map typeMap = new HashMap<>(); typeMap.put(ByteBufferUtil.bytes("c"), ListType.getInstance(Int32Type.instance, true)); - CompoundSparseCellNameType.WithCollection nameType = new CompoundSparseCellNameType.WithCollection(Collections.EMPTY_LIST, ColumnToCollectionType.getInstance(typeMap)); + ColumnDefinition definition = new ColumnDefinition(cfm, ByteBufferUtil.bytes("c"), ListType.getInstance(Int32Type.instance, true), 0, ColumnDefinition.Kind.REGULAR); List cells = new ArrayList<>(columnValues.size()); @@ -172,7 +166,7 @@ public class ColumnConditionTest { ByteBuffer key = Int32Serializer.instance.serialize(i); ByteBuffer value = columnValues.get(i); - cells.add(new BufferCell(nameType.create(Composites.EMPTY, definition, key), value)); + cells.add(new TestCell(definition, CellPath.create(key), value, LivenessInfo.NONE)); }; } @@ -183,7 +177,7 @@ public class ColumnConditionTest // sets use the same check as lists public void testListCollectionBoundAppliesTo() throws InvalidRequestException { - ColumnDefinition definition = new ColumnDefinition("ks", "cf", new ColumnIdentifier("c", true), ListType.getInstance(Int32Type.instance, true), null, null, null, null, null); + ColumnDefinition definition = ColumnDefinition.regularDef("ks", "cf", "c", ListType.getInstance(Int32Type.instance, true), null); // EQ ColumnCondition condition = ColumnCondition.condition(definition, null, new Lists.Value(Arrays.asList(ONE)), Operator.EQ); @@ -294,7 +288,6 @@ public class ColumnConditionTest CFMetaData cfm = CFMetaData.compile("create table foo(a int PRIMARY KEY, b int, c set)", "ks"); Map typeMap = new HashMap<>(); typeMap.put(ByteBufferUtil.bytes("c"), SetType.getInstance(Int32Type.instance, true)); - CompoundSparseCellNameType.WithCollection nameType = new CompoundSparseCellNameType.WithCollection(Collections.EMPTY_LIST, ColumnToCollectionType.getInstance(typeMap)); ColumnDefinition definition = new ColumnDefinition(cfm, ByteBufferUtil.bytes("c"), SetType.getInstance(Int32Type.instance, true), 0, ColumnDefinition.Kind.REGULAR); List cells = new ArrayList<>(columnValues.size()); @@ -303,7 +296,7 @@ public class ColumnConditionTest for (int i = 0; i < columnValues.size(); i++) { ByteBuffer key = columnValues.get(i); - cells.add(new BufferCell(nameType.create(Composites.EMPTY, definition, key), ByteBufferUtil.EMPTY_BYTE_BUFFER)); + cells.add(new TestCell(definition, CellPath.create(key), ByteBufferUtil.EMPTY_BYTE_BUFFER, LivenessInfo.NONE)); }; } @@ -313,7 +306,7 @@ public class ColumnConditionTest @Test public void testSetCollectionBoundAppliesTo() throws InvalidRequestException { - ColumnDefinition definition = new ColumnDefinition("ks", "cf", new ColumnIdentifier("c", true), SetType.getInstance(Int32Type.instance, true), null, null, null, null, null); + ColumnDefinition definition = ColumnDefinition.regularDef("ks", "cf", "c", ListType.getInstance(Int32Type.instance, true), null); // EQ ColumnCondition condition = ColumnCondition.condition(definition, null, new Sets.Value(set(ONE)), Operator.EQ); @@ -427,14 +420,13 @@ public class ColumnConditionTest CFMetaData cfm = CFMetaData.compile("create table foo(a int PRIMARY KEY, b map)", "ks"); Map typeMap = new HashMap<>(); typeMap.put(ByteBufferUtil.bytes("b"), MapType.getInstance(Int32Type.instance, Int32Type.instance, true)); - CompoundSparseCellNameType.WithCollection nameType = new CompoundSparseCellNameType.WithCollection(Collections.EMPTY_LIST, ColumnToCollectionType.getInstance(typeMap)); ColumnDefinition definition = new ColumnDefinition(cfm, ByteBufferUtil.bytes("b"), MapType.getInstance(Int32Type.instance, Int32Type.instance, true), 0, ColumnDefinition.Kind.REGULAR); List cells = new ArrayList<>(columnValues.size()); if (columnValues != null) { for (Map.Entry entry : columnValues.entrySet()) - cells.add(new BufferCell(nameType.create(Composites.EMPTY, definition, entry.getKey()), entry.getValue())); + cells.add(new TestCell(definition, CellPath.create(entry.getKey()), entry.getValue(), LivenessInfo.NONE)); } return bound.mapAppliesTo(MapType.getInstance(Int32Type.instance, Int32Type.instance, true), cells.iterator(), conditionValues, bound.operator); @@ -443,7 +435,7 @@ public class ColumnConditionTest @Test public void testMapCollectionBoundIsSatisfiedByValue() throws InvalidRequestException { - ColumnDefinition definition = new ColumnDefinition("ks", "cf", new ColumnIdentifier("b", true), MapType.getInstance(Int32Type.instance, Int32Type.instance, true), null, null, null, null, null); + ColumnDefinition definition = ColumnDefinition.regularDef("ks", "cf", "c", ListType.getInstance(Int32Type.instance, true), null); Map placeholderMap = new TreeMap<>(); placeholderMap.put(ONE, ONE); @@ -581,4 +573,45 @@ public class ColumnConditionTest assertTrue(mapAppliesTo(bound, map(ByteBufferUtil.EMPTY_BYTE_BUFFER, ONE), map(ByteBufferUtil.EMPTY_BYTE_BUFFER, ONE))); assertTrue(mapAppliesTo(bound, map(ONE, ByteBufferUtil.EMPTY_BYTE_BUFFER), map(ONE, ByteBufferUtil.EMPTY_BYTE_BUFFER))); } -} \ No newline at end of file + + static class TestCell extends AbstractCell + { + private final ColumnDefinition column; + private final CellPath path; + private final ByteBuffer value; + private final LivenessInfo info; + + public TestCell(ColumnDefinition column, CellPath path, ByteBuffer value, LivenessInfo info) + { + this.column = column; + this.path = path; + this.value = value; + this.info = info.takeAlias(); + } + + public ColumnDefinition column() + { + return column; + } + + public boolean isCounterCell() + { + return false; + } + + public ByteBuffer value() + { + return value; + } + + public LivenessInfo livenessInfo() + { + return info; + } + + public CellPath path() + { + return path; + } + } +} diff --git a/test/unit/org/apache/cassandra/cql3/DeleteTest.java b/test/unit/org/apache/cassandra/cql3/DeleteTest.java index 812d729f5b..0b8858629d 100644 --- a/test/unit/org/apache/cassandra/cql3/DeleteTest.java +++ b/test/unit/org/apache/cassandra/cql3/DeleteTest.java @@ -18,6 +18,11 @@ package org.apache.cassandra.cql3; +import org.junit.AfterClass; +import org.junit.Assert; +import org.junit.BeforeClass; +import org.junit.Test; + import com.datastax.driver.core.Cluster; import com.datastax.driver.core.PreparedStatement; import com.datastax.driver.core.ResultSetFuture; @@ -26,10 +31,6 @@ import org.apache.cassandra.SchemaLoader; import org.apache.cassandra.config.DatabaseDescriptor; import org.apache.cassandra.config.Schema; import org.apache.cassandra.service.EmbeddedCassandraService; -import org.junit.Assert; -import org.junit.AfterClass; -import org.junit.BeforeClass; -import org.junit.Test; public class DeleteTest extends SchemaLoader { diff --git a/test/unit/org/apache/cassandra/cql3/IndexQueryPagingTest.java b/test/unit/org/apache/cassandra/cql3/IndexQueryPagingTest.java new file mode 100644 index 0000000000..45994c7301 --- /dev/null +++ b/test/unit/org/apache/cassandra/cql3/IndexQueryPagingTest.java @@ -0,0 +1,88 @@ +package org.apache.cassandra.cql3; + +import org.junit.Test; + +import com.datastax.driver.core.Session; +import com.datastax.driver.core.SimpleStatement; +import com.datastax.driver.core.Statement; + +import static org.junit.Assert.assertEquals; + +import org.apache.cassandra.config.DatabaseDescriptor; +import org.apache.cassandra.dht.Murmur3Partitioner; + +public class IndexQueryPagingTest extends CQLTester +{ + /* + * Some simple tests to verify the behaviour of paging during + * 2i queries. We only use a single index type (CompositesIndexOnRegular) + * as the code we want to exercise here is in their abstract + * base class. + */ + + @Test + public void pagingOnRegularColumn() throws Throwable + { + createTable("CREATE TABLE %s (" + + " k1 int," + + " v1 int," + + "PRIMARY KEY (k1))"); + createIndex("CREATE INDEX ON %s(v1)"); + + int rowCount = 3; + for (int i=0; i=0 AND c1<=3 AND v1=0", rowCount); + } + + private void executePagingQuery(String cql, int rowCount) + { + // Execute an index query which should return all rows, + // setting the fetch size < than the row count. Assert + // that all rows are returned, so we know that paging + // of the results was involved. + Session session = sessionNet(maxProtocolVersion); + Statement stmt = new SimpleStatement(String.format(cql, KEYSPACE + "." + currentTable())); + stmt.setFetchSize(rowCount - 1); + assertEquals(rowCount, session.execute(stmt).all().size()); + } +} diff --git a/test/unit/org/apache/cassandra/cql3/NonNativeTimestampTest.java b/test/unit/org/apache/cassandra/cql3/NonNativeTimestampTest.java index 80c5e3b33c..37dc560495 100644 --- a/test/unit/org/apache/cassandra/cql3/NonNativeTimestampTest.java +++ b/test/unit/org/apache/cassandra/cql3/NonNativeTimestampTest.java @@ -23,63 +23,31 @@ import java.nio.charset.CharacterCodingException; import java.util.Arrays; import java.util.Collections; -import org.junit.BeforeClass; import org.junit.Test; -import org.apache.cassandra.SchemaLoader; -import org.apache.cassandra.config.Schema; -import org.apache.cassandra.db.ConsistencyLevel; -import org.apache.cassandra.exceptions.RequestExecutionException; -import org.apache.cassandra.exceptions.RequestValidationException; -import org.apache.cassandra.service.EmbeddedCassandraService; -import org.apache.cassandra.service.QueryState; -import org.apache.cassandra.utils.ByteBufferUtil; - import static junit.framework.Assert.assertEquals; import static junit.framework.Assert.assertFalse; import static org.junit.Assert.assertTrue; -public class NonNativeTimestampTest extends SchemaLoader +public class NonNativeTimestampTest extends CQLTester { - @BeforeClass - public static void setup() throws Exception - { - Schema.instance.clear(); - EmbeddedCassandraService cassandra = new EmbeddedCassandraService(); - cassandra.start(); - } - @Test - public void setServerTimestampForNonCqlNativeStatements() throws RequestValidationException, RequestExecutionException + public void setServerTimestampForNonCqlNativeStatements() throws Throwable { - String createKsCQL = "CREATE KEYSPACE non_native_ts_test" + - " WITH REPLICATION = { 'class' : 'SimpleStrategy', 'replication_factor' : 1 };"; - String createTableCQL = "CREATE TABLE non_native_ts_test.table_0 (k int PRIMARY KEY, v int)"; - String insertCQL = "INSERT INTO non_native_ts_test.table_0 (k, v) values (1, ?)"; - String selectCQL = "SELECT v, writetime(v) AS wt FROM non_native_ts_test.table_0 WHERE k = 1"; + createTable("CREATE TABLE %s (k int PRIMARY KEY, v int)"); - QueryProcessor.instance.process(createKsCQL, - QueryState.forInternalCalls(), - QueryOptions.forInternalCalls(Collections.emptyList())); - QueryProcessor.instance.process(createTableCQL, - QueryState.forInternalCalls(), - QueryOptions.forInternalCalls(Collections.emptyList())); - QueryProcessor.instance.process(insertCQL, - QueryState.forInternalCalls(), - QueryOptions.forInternalCalls(ConsistencyLevel.ONE, - Arrays.asList(ByteBufferUtil.bytes(2)))); - UntypedResultSet.Row row = QueryProcessor.instance.executeInternal(selectCQL).one(); + execute("INSERT INTO %s (k, v) values (1, ?)", 2); + + UntypedResultSet.Row row = execute("SELECT v, writetime(v) AS wt FROM %s WHERE k = 1").one(); assertEquals(2, row.getInt("v")); long timestamp1 = row.getLong("wt"); assertFalse(timestamp1 == -1l); // per CASSANDRA-8246 the two updates will have the same (incorrect) // timestamp, so reconcilliation is by value and the "older" update wins - QueryProcessor.instance.process(insertCQL, - QueryState.forInternalCalls(), - QueryOptions.forInternalCalls(ConsistencyLevel.ONE, - Arrays.asList(ByteBufferUtil.bytes(1)))); - row = QueryProcessor.executeInternal(selectCQL).one(); + execute("INSERT INTO %s (k, v) values (1, ?)", 1); + + row = execute("SELECT v, writetime(v) AS wt FROM %s WHERE k = 1").one(); assertEquals(1, row.getInt("v")); assertTrue(row.getLong("wt") > timestamp1); } diff --git a/test/unit/org/apache/cassandra/cql3/SimpleQueryTest.java b/test/unit/org/apache/cassandra/cql3/SimpleQueryTest.java new file mode 100644 index 0000000000..ad0dd7be88 --- /dev/null +++ b/test/unit/org/apache/cassandra/cql3/SimpleQueryTest.java @@ -0,0 +1,532 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.cassandra.cql3; + +import java.util.*; +import org.junit.Test; + +import static junit.framework.Assert.*; + +public class SimpleQueryTest extends CQLTester +{ + @Test + public void testStaticCompactTables() throws Throwable + { + createTable("CREATE TABLE %s (k text PRIMARY KEY, v1 int, v2 text) WITH COMPACT STORAGE"); + + execute("INSERT INTO %s (k, v1, v2) values (?, ?, ?)", "first", 1, "value1"); + execute("INSERT INTO %s (k, v1, v2) values (?, ?, ?)", "second", 2, "value2"); + execute("INSERT INTO %s (k, v1, v2) values (?, ?, ?)", "third", 3, "value3"); + + assertRows(execute("SELECT * FROM %s WHERE k = ?", "first"), + row("first", 1, "value1") + ); + + assertRows(execute("SELECT v2 FROM %s WHERE k = ?", "second"), + row("value2") + ); + + // Murmur3 order + assertRows(execute("SELECT * FROM %s"), + row("third", 3, "value3"), + row("second", 2, "value2"), + row("first", 1, "value1") + ); + } + + @Test + public void testDynamicCompactTables() throws Throwable + { + createTable("CREATE TABLE %s (k text, t int, v text, PRIMARY KEY (k, t));"); + + execute("INSERT INTO %s (k, t, v) values (?, ?, ?)", "key", 1, "v11"); + execute("INSERT INTO %s (k, t, v) values (?, ?, ?)", "key", 2, "v12"); + execute("INSERT INTO %s (k, t, v) values (?, ?, ?)", "key", 3, "v13"); + + flush(); + + execute("INSERT INTO %s (k, t, v) values (?, ?, ?)", "key", 4, "v14"); + execute("INSERT INTO %s (k, t, v) values (?, ?, ?)", "key", 5, "v15"); + + assertRows(execute("SELECT * FROM %s"), + row("key", 1, "v11"), + row("key", 2, "v12"), + row("key", 3, "v13"), + row("key", 4, "v14"), + row("key", 5, "v15") + ); + + assertRows(execute("SELECT * FROM %s WHERE k = ? AND t > ?", "key", 3), + row("key", 4, "v14"), + row("key", 5, "v15") + ); + + assertRows(execute("SELECT * FROM %s WHERE k = ? AND t >= ? AND t < ?", "key", 2, 4), + row("key", 2, "v12"), + row("key", 3, "v13") + ); + + // Reversed queries + + assertRows(execute("SELECT * FROM %s WHERE k = ? ORDER BY t DESC", "key"), + row("key", 5, "v15"), + row("key", 4, "v14"), + row("key", 3, "v13"), + row("key", 2, "v12"), + row("key", 1, "v11") + ); + + assertRows(execute("SELECT * FROM %s WHERE k = ? AND t > ? ORDER BY t DESC", "key", 3), + row("key", 5, "v15"), + row("key", 4, "v14") + ); + + assertRows(execute("SELECT * FROM %s WHERE k = ? AND t >= ? AND t < ? ORDER BY t DESC", "key", 2, 4), + row("key", 3, "v13"), + row("key", 2, "v12") + ); + } + + @Test + public void testTableWithoutClustering() throws Throwable + { + createTable("CREATE TABLE %s (k text PRIMARY KEY, v1 int, v2 text);"); + + execute("INSERT INTO %s (k, v1, v2) values (?, ?, ?)", "first", 1, "value1"); + execute("INSERT INTO %s (k, v1, v2) values (?, ?, ?)", "second", 2, "value2"); + execute("INSERT INTO %s (k, v1, v2) values (?, ?, ?)", "third", 3, "value3"); + + flush(); + + assertRows(execute("SELECT * FROM %s WHERE k = ?", "first"), + row("first", 1, "value1") + ); + + assertRows(execute("SELECT v2 FROM %s WHERE k = ?", "second"), + row("value2") + ); + + assertRows(execute("SELECT * FROM %s"), + row("third", 3, "value3"), + row("second", 2, "value2"), + row("first", 1, "value1") + ); + } + + @Test + public void testTableWithOneClustering() throws Throwable + { + createTable("CREATE TABLE %s (k text, t int, v1 text, v2 text, PRIMARY KEY (k, t));"); + + execute("INSERT INTO %s (k, t, v1, v2) values (?, ?, ?, ?)", "key", 1, "v11", "v21"); + execute("INSERT INTO %s (k, t, v1, v2) values (?, ?, ?, ?)", "key", 2, "v12", "v22"); + execute("INSERT INTO %s (k, t, v1, v2) values (?, ?, ?, ?)", "key", 3, "v13", "v23"); + + flush(); + + execute("INSERT INTO %s (k, t, v1, v2) values (?, ?, ?, ?)", "key", 4, "v14", "v24"); + execute("INSERT INTO %s (k, t, v1, v2) values (?, ?, ?, ?)", "key", 5, "v15", "v25"); + + assertRows(execute("SELECT * FROM %s"), + row("key", 1, "v11", "v21"), + row("key", 2, "v12", "v22"), + row("key", 3, "v13", "v23"), + row("key", 4, "v14", "v24"), + row("key", 5, "v15", "v25") + ); + + assertRows(execute("SELECT * FROM %s WHERE k = ? AND t > ?", "key", 3), + row("key", 4, "v14", "v24"), + row("key", 5, "v15", "v25") + ); + + assertRows(execute("SELECT * FROM %s WHERE k = ? AND t >= ? AND t < ?", "key", 2, 4), + row("key", 2, "v12", "v22"), + row("key", 3, "v13", "v23") + ); + + // Reversed queries + + assertRows(execute("SELECT * FROM %s WHERE k = ? ORDER BY t DESC", "key"), + row("key", 5, "v15", "v25"), + row("key", 4, "v14", "v24"), + row("key", 3, "v13", "v23"), + row("key", 2, "v12", "v22"), + row("key", 1, "v11", "v21") + ); + + assertRows(execute("SELECT * FROM %s WHERE k = ? AND t > ? ORDER BY t DESC", "key", 3), + row("key", 5, "v15", "v25"), + row("key", 4, "v14", "v24") + ); + + assertRows(execute("SELECT * FROM %s WHERE k = ? AND t >= ? AND t < ? ORDER BY t DESC", "key", 2, 4), + row("key", 3, "v13", "v23"), + row("key", 2, "v12", "v22") + ); + } + + @Test + public void testTableWithReverseClusteringOrder() throws Throwable + { + createTable("CREATE TABLE %s (k text, t int, v1 text, v2 text, PRIMARY KEY (k, t)) WITH CLUSTERING ORDER BY (t DESC);"); + + execute("INSERT INTO %s (k, t, v1, v2) values (?, ?, ?, ?)", "key", 1, "v11", "v21"); + execute("INSERT INTO %s (k, t, v1, v2) values (?, ?, ?, ?)", "key", 2, "v12", "v22"); + execute("INSERT INTO %s (k, t, v1, v2) values (?, ?, ?, ?)", "key", 3, "v13", "v23"); + + flush(); + + execute("INSERT INTO %s (k, t, v1, v2) values (?, ?, ?, ?)", "key", 4, "v14", "v24"); + execute("INSERT INTO %s (k, t, v1, v2) values (?, ?, ?, ?)", "key", 5, "v15", "v25"); + + assertRows(execute("SELECT * FROM %s"), + row("key", 5, "v15", "v25"), + row("key", 4, "v14", "v24"), + row("key", 3, "v13", "v23"), + row("key", 2, "v12", "v22"), + row("key", 1, "v11", "v21") + ); + + assertRows(execute("SELECT * FROM %s WHERE k = ? ORDER BY t ASC", "key"), + row("key", 1, "v11", "v21"), + row("key", 2, "v12", "v22"), + row("key", 3, "v13", "v23"), + row("key", 4, "v14", "v24"), + row("key", 5, "v15", "v25") + ); + + assertRows(execute("SELECT * FROM %s WHERE k = ? AND t > ?", "key", 3), + row("key", 5, "v15", "v25"), + row("key", 4, "v14", "v24") + ); + + assertRows(execute("SELECT * FROM %s WHERE k = ? AND t >= ? AND t < ?", "key", 2, 4), + row("key", 3, "v13", "v23"), + row("key", 2, "v12", "v22") + ); + + // Reversed queries + + assertRows(execute("SELECT * FROM %s WHERE k = ? ORDER BY t DESC", "key"), + row("key", 5, "v15", "v25"), + row("key", 4, "v14", "v24"), + row("key", 3, "v13", "v23"), + row("key", 2, "v12", "v22"), + row("key", 1, "v11", "v21") + ); + + assertRows(execute("SELECT * FROM %s WHERE k = ? AND t > ? ORDER BY t DESC", "key", 3), + row("key", 5, "v15", "v25"), + row("key", 4, "v14", "v24") + ); + + assertRows(execute("SELECT * FROM %s WHERE k = ? AND t >= ? AND t < ? ORDER BY t DESC", "key", 2, 4), + row("key", 3, "v13", "v23"), + row("key", 2, "v12", "v22") + ); + } + + @Test + public void testTableWithTwoClustering() throws Throwable + { + createTable("CREATE TABLE %s (k text, t1 text, t2 int, v text, PRIMARY KEY (k, t1, t2));"); + + execute("INSERT INTO %s (k, t1, t2, v) values (?, ?, ?, ?)", "key", "v1", 1, "v1"); + execute("INSERT INTO %s (k, t1, t2, v) values (?, ?, ?, ?)", "key", "v1", 2, "v2"); + execute("INSERT INTO %s (k, t1, t2, v) values (?, ?, ?, ?)", "key", "v2", 1, "v3"); + execute("INSERT INTO %s (k, t1, t2, v) values (?, ?, ?, ?)", "key", "v2", 2, "v4"); + execute("INSERT INTO %s (k, t1, t2, v) values (?, ?, ?, ?)", "key", "v2", 3, "v5"); + flush(); + + assertRows(execute("SELECT * FROM %s"), + row("key", "v1", 1, "v1"), + row("key", "v1", 2, "v2"), + row("key", "v2", 1, "v3"), + row("key", "v2", 2, "v4"), + row("key", "v2", 3, "v5") + ); + + assertRows(execute("SELECT * FROM %s WHERE k = ? AND t1 >= ?", "key", "v2"), + row("key", "v2", 1, "v3"), + row("key", "v2", 2, "v4"), + row("key", "v2", 3, "v5") + ); + + assertRows(execute("SELECT * FROM %s WHERE k = ? AND t1 >= ? ORDER BY t1 DESC", "key", "v2"), + row("key", "v2", 3, "v5"), + row("key", "v2", 2, "v4"), + row("key", "v2", 1, "v3") + ); + } + + @Test + public void testTableWithLargePartition() throws Throwable + { + createTable("CREATE TABLE %s (k text, t1 int, t2 int, v text, PRIMARY KEY (k, t1, t2));"); + + for (int t1 = 0; t1 < 20; t1++) + for (int t2 = 0; t2 < 10; t2++) + execute("INSERT INTO %s (k, t1, t2, v) values (?, ?, ?, ?)", "key", t1, t2, "someSemiLargeTextForValue_" + t1 + "_" + t2); + + flush(); + + Object[][] expected = new Object[10][]; + for (int t2 = 0; t2 < 10; t2++) + expected[t2] = row("key", 15, t2); + + assertRows(execute("SELECT k, t1, t2 FROM %s WHERE k=? AND t1=?", "key", 15), expected); + + Object[][] expectedReverse = new Object[10][]; + for (int t2 = 9; t2 >= 0; t2--) + expectedReverse[9 - t2] = row("key", 15, t2); + + assertRows(execute("SELECT k, t1, t2 FROM %s WHERE k=? AND t1=? ORDER BY t1 DESC, t2 DESC", "key", 15), expectedReverse); + } + + @Test + public void testRowDeletion() throws Throwable + { + int N = 4; + + createTable("CREATE TABLE %s (k text, t int, v1 text, v2 int, PRIMARY KEY (k, t));"); + + for (int t = 0; t < N; t++) + execute("INSERT INTO %s (k, t, v1, v2) values (?, ?, ?, ?)", "key", t, "v" + t, t + 10); + + flush(); + + for (int i = 0; i < N / 2; i++) + execute("DELETE FROM %s WHERE k=? AND t=?", "key", i * 2); + + Object[][] expected = new Object[N/2][]; + for (int i = 0; i < N / 2; i++) + { + int t = i * 2 + 1; + expected[i] = row("key", t, "v" + t, t + 10); + } + + assertRows(execute("SELECT * FROM %s"), expected); + } + + @Test + public void testRangeTombstones() throws Throwable + { + int N = 100; + + createTable("CREATE TABLE %s (k text, t1 int, t2 int, v text, PRIMARY KEY (k, t1, t2));"); + + for (int t1 = 0; t1 < 3; t1++) + for (int t2 = 0; t2 < N; t2++) + execute("INSERT INTO %s (k, t1, t2, v) values (?, ?, ?, ?)", "key", t1, t2, "someSemiLargeTextForValue_" + t1 + "_" + t2); + + flush(); + + execute("DELETE FROM %s WHERE k=? AND t1=?", "key", 1); + + flush(); + + Object[][] expected = new Object[2*N][]; + for (int t2 = 0; t2 < N; t2++) + { + expected[t2] = row("key", 0, t2, "someSemiLargeTextForValue_0_" + t2); + expected[N + t2] = row("key", 2, t2, "someSemiLargeTextForValue_2_" + t2); + } + + assertRows(execute("SELECT * FROM %s"), expected); + } + + @Test + public void test2ndaryIndexes() throws Throwable + { + createTable("CREATE TABLE %s (k text, t int, v text, PRIMARY KEY (k, t));"); + + execute("CREATE INDEX ON %s(v)"); + + execute("INSERT INTO %s (k, t, v) values (?, ?, ?)", "key1", 1, "foo"); + execute("INSERT INTO %s (k, t, v) values (?, ?, ?)", "key1", 2, "bar"); + execute("INSERT INTO %s (k, t, v) values (?, ?, ?)", "key2", 1, "foo"); + + flush(); + + execute("INSERT INTO %s (k, t, v) values (?, ?, ?)", "key2", 2, "foo"); + execute("INSERT INTO %s (k, t, v) values (?, ?, ?)", "key2", 3, "bar"); + + assertRows(execute("SELECT * FROM %s WHERE v = ?", "foo"), + row("key1", 1, "foo"), + row("key2", 1, "foo"), + row("key2", 2, "foo") + ); + + assertRows(execute("SELECT * FROM %s WHERE v = ?", "bar"), + row("key1", 2, "bar"), + row("key2", 3, "bar") + ); + } + + @Test + public void testStaticColumns() throws Throwable + { + createTable("CREATE TABLE %s (k text, t int, s text static, v text, PRIMARY KEY (k, t));"); + + execute("INSERT INTO %s (k, t, v, s) values (?, ?, ?, ?)", "key1", 1, "foo1", "st1"); + execute("INSERT INTO %s (k, t, v, s) values (?, ?, ?, ?)", "key1", 2, "foo2", "st2"); + + flush(); + + execute("INSERT INTO %s (k, t, v, s) values (?, ?, ?, ?)", "key1", 3, "foo3", "st3"); + execute("INSERT INTO %s (k, t, v) values (?, ?, ?)", "key1", 4, "foo4"); + + assertRows(execute("SELECT * FROM %s"), + row("key1", 1, "st3", "foo1"), + row("key1", 2, "st3", "foo2"), + row("key1", 3, "st3", "foo3"), + row("key1", 4, "st3", "foo4") + ); + + assertRows(execute("SELECT s FROM %s WHERE k = ?", "key1"), + row("st3"), + row("st3"), + row("st3"), + row("st3") + ); + + assertRows(execute("SELECT DISTINCT s FROM %s WHERE k = ?", "key1"), + row("st3") + ); + } + + @Test + public void testDistinct() throws Throwable + { + createTable("CREATE TABLE %s (k text, t int, v text, PRIMARY KEY (k, t));"); + + execute("INSERT INTO %s (k, t, v) values (?, ?, ?)", "key1", 1, "foo1"); + execute("INSERT INTO %s (k, t, v) values (?, ?, ?)", "key1", 2, "foo2"); + + flush(); + + execute("INSERT INTO %s (k, t, v) values (?, ?, ?)", "key1", 3, "foo3"); + execute("INSERT INTO %s (k, t, v) values (?, ?, ?)", "key2", 4, "foo4"); + execute("INSERT INTO %s (k, t, v) values (?, ?, ?)", "key2", 5, "foo5"); + + assertRows(execute("SELECT DISTINCT k FROM %s"), + row("key1"), + row("key2") + ); + } + + @Test + public void collectionDeletionTest() throws Throwable + { + createTable("CREATE TABLE %s (k text PRIMARY KEY, s set);"); + + execute("INSERT INTO %s (k, s) VALUES (?, ?)", 1, set(1)); + + flush(); + + execute("INSERT INTO %s (k, s) VALUES (?, ?)", 1, set(2)); + + assertRows(execute("SELECT s FROM %s WHERE k = ?", 1), + row(set(2)) + ); + } + + @Test + public void limitWithMultigetTest() throws Throwable + { + createTable("CREATE TABLE %s (k int PRIMARY KEY, v int);"); + + execute("INSERT INTO %s (k, v) VALUES (?, ?)", 0, 0); + execute("INSERT INTO %s (k, v) VALUES (?, ?)", 1, 1); + execute("INSERT INTO %s (k, v) VALUES (?, ?)", 2, 2); + execute("INSERT INTO %s (k, v) VALUES (?, ?)", 3, 3); + + assertRows(execute("SELECT v FROM %s WHERE k IN ? LIMIT ?", list(0, 1, 2, 3), 2), + row(0), + row(1) + ); + } + + @Test + public void staticDistinctTest() throws Throwable + { + createTable("CREATE TABLE %s ( k int, p int, s int static, PRIMARY KEY (k, p))"); + + execute("INSERT INTO %s (k, p) VALUES (?, ?)", 1, 1); + execute("INSERT INTO %s (k, p) VALUES (?, ?)", 1, 2); + + assertRows(execute("SELECT k, s FROM %s"), + row(1, null), + row(1, null) + ); + assertRows(execute("SELECT DISTINCT k, s FROM %s"), + row(1, null) + ); + assertRows(execute("SELECT DISTINCT s FROM %s WHERE k=?", 1), + row((Object)null) + ); + assertEmpty(execute("SELECT DISTINCT s FROM %s WHERE k=?", 2)); + } + + @Test + public void testCompactStorageUpdateWithNull() throws Throwable + { + createTable("CREATE TABLE %s (partitionKey int," + + "clustering_1 int," + + "value int," + + " PRIMARY KEY (partitionKey, clustering_1)) WITH COMPACT STORAGE"); + + execute("INSERT INTO %s (partitionKey, clustering_1, value) VALUES (0, 0, 0)"); + execute("INSERT INTO %s (partitionKey, clustering_1, value) VALUES (0, 1, 1)"); + + flush(); + + execute("UPDATE %s SET value = ? WHERE partitionKey = ? AND clustering_1 = ?", null, 0, 0); + + assertRows(execute("SELECT * FROM %s WHERE partitionKey = ? AND (clustering_1) IN ((?), (?))", 0, 0, 1), + row(0, 1, 1) + ); + } + + @Test + public void test2ndaryIndexBug() throws Throwable + { + createTable("CREATE TABLE %s (k int, c1 int, c2 int, v int, PRIMARY KEY(k, c1, c2))"); + + execute("CREATE INDEX v_idx ON %s(v)"); + + execute("INSERT INTO %s (k, c1, c2, v) VALUES (?, ?, ?, ?)", 0, 0, 0, 0); + execute("INSERT INTO %s (k, c1, c2, v) VALUES (?, ?, ?, ?)", 0, 1, 0, 0); + + assertRows(execute("SELECT * FROM %s WHERE v=?", 0), + row(0, 0, 0, 0), + row(0, 1, 0, 0) + ); + + flush(); + + execute("DELETE FROM %s WHERE k=? AND c1=?", 0, 1); + + flush(); + + assertRows(execute("SELECT * FROM %s WHERE v=?", 0), + row(0, 0, 0, 0) + ); + } +} diff --git a/test/unit/org/apache/cassandra/cql3/ThriftCompatibilityTest.java b/test/unit/org/apache/cassandra/cql3/ThriftCompatibilityTest.java index 7b72ef867f..17e50b343b 100644 --- a/test/unit/org/apache/cassandra/cql3/ThriftCompatibilityTest.java +++ b/test/unit/org/apache/cassandra/cql3/ThriftCompatibilityTest.java @@ -17,14 +17,15 @@ */ package org.apache.cassandra.cql3; -import org.apache.cassandra.config.KSMetaData; -import org.apache.cassandra.db.marshal.Int32Type; -import org.apache.cassandra.locator.SimpleStrategy; -import org.apache.cassandra.utils.ByteBufferUtil; import org.junit.BeforeClass; import org.junit.Test; import org.apache.cassandra.SchemaLoader; +import org.apache.cassandra.config.KSMetaData; +import org.apache.cassandra.config.Schema; +import org.apache.cassandra.db.marshal.Int32Type; +import org.apache.cassandra.locator.SimpleStrategy; +import org.apache.cassandra.utils.ByteBufferUtil; import static org.junit.Assert.assertEquals; import static org.junit.Assert.fail; @@ -36,10 +37,10 @@ public class ThriftCompatibilityTest extends SchemaLoader { // The before class annotation of SchemaLoader will prepare the service so no need to do it here SchemaLoader.createKeyspace("thriftcompat", - SimpleStrategy.class, - KSMetaData.optsWithRF(1), - jdbcSparseCFMD("thriftcompat", "JdbcInteger", Int32Type.instance) - .addColumnDefinition(integerColumn("thriftcompat", "JdbcInteger"))); + SimpleStrategy.class, + KSMetaData.optsWithRF(1), + SchemaLoader.jdbcCFMD("thriftcompat", "JdbcInteger", Int32Type.instance) + .addColumnDefinition(integerColumn("thriftcompat", "JdbcInteger"))); } private static UntypedResultSet execute(String query) diff --git a/test/unit/org/apache/cassandra/cql3/restrictions/PrimaryKeyRestrictionSetTest.java b/test/unit/org/apache/cassandra/cql3/restrictions/PrimaryKeyRestrictionSetTest.java index 05d6e98afe..d738c46935 100644 --- a/test/unit/org/apache/cassandra/cql3/restrictions/PrimaryKeyRestrictionSetTest.java +++ b/test/unit/org/apache/cassandra/cql3/restrictions/PrimaryKeyRestrictionSetTest.java @@ -18,9 +18,9 @@ package org.apache.cassandra.cql3.restrictions; import java.nio.ByteBuffer; -import java.util.ArrayList; -import java.util.List; +import java.util.*; +import com.google.common.collect.Iterables; import org.junit.Test; import org.apache.cassandra.config.CFMetaData; @@ -28,11 +28,7 @@ import org.apache.cassandra.config.ColumnDefinition; import org.apache.cassandra.cql3.*; import org.apache.cassandra.cql3.Term.MultiItemTerminal; import org.apache.cassandra.cql3.statements.Bound; -import org.apache.cassandra.db.ColumnFamilyType; -import org.apache.cassandra.db.composites.Composite; -import org.apache.cassandra.db.composites.Composite.EOC; -import org.apache.cassandra.db.composites.Composites; -import org.apache.cassandra.db.composites.CompoundSparseCellNameType; +import org.apache.cassandra.db.*; import org.apache.cassandra.db.marshal.AbstractType; import org.apache.cassandra.db.marshal.Int32Type; import org.apache.cassandra.exceptions.InvalidRequestException; @@ -45,72 +41,72 @@ import static org.junit.Assert.assertTrue; public class PrimaryKeyRestrictionSetTest { @Test - public void testBoundsAsCompositesWithNoRestrictions() throws InvalidRequestException + public void testboundsAsClusteringWithNoRestrictions() throws InvalidRequestException { CFMetaData cfMetaData = newCFMetaData(1); - PrimaryKeyRestrictions restrictions = new PrimaryKeyRestrictionSet(cfMetaData.comparator); + PrimaryKeyRestrictions restrictions = new PrimaryKeyRestrictionSet(cfMetaData.comparator, false); - List bounds = restrictions.boundsAsComposites(Bound.START, QueryOptions.DEFAULT); + SortedSet bounds = restrictions.boundsAsClustering(Bound.START, QueryOptions.DEFAULT); assertEquals(1, bounds.size()); - assertEmptyComposite(bounds.get(0)); + assertEmptyStart(get(bounds, 0)); - bounds = restrictions.boundsAsComposites(Bound.END, QueryOptions.DEFAULT); + bounds = restrictions.boundsAsClustering(Bound.END, QueryOptions.DEFAULT); assertEquals(1, bounds.size()); - assertEmptyComposite(bounds.get(0)); + assertEmptyEnd(get(bounds, 0)); } /** * Test 'clustering_0 = 1' with only one clustering column */ @Test - public void testBoundsAsCompositesWithOneEqRestrictionsAndOneClusteringColumn() throws InvalidRequestException + public void testboundsAsClusteringWithOneEqRestrictionsAndOneClusteringColumn() throws InvalidRequestException { CFMetaData cfMetaData = newCFMetaData(1); ByteBuffer clustering_0 = ByteBufferUtil.bytes(1); Restriction eq = newSingleEq(cfMetaData, 0, clustering_0); - PrimaryKeyRestrictions restrictions = new PrimaryKeyRestrictionSet(cfMetaData.comparator); + PrimaryKeyRestrictions restrictions = new PrimaryKeyRestrictionSet(cfMetaData.comparator, false); restrictions = restrictions.mergeWith(eq); - List bounds = restrictions.boundsAsComposites(Bound.START, QueryOptions.DEFAULT); + SortedSet bounds = restrictions.boundsAsClustering(Bound.START, QueryOptions.DEFAULT); assertEquals(1, bounds.size()); - assertComposite(bounds.get(0), clustering_0, EOC.START); + assertStartBound(get(bounds, 0), true, clustering_0); - bounds = restrictions.boundsAsComposites(Bound.END, QueryOptions.DEFAULT); + bounds = restrictions.boundsAsClustering(Bound.END, QueryOptions.DEFAULT); assertEquals(1, bounds.size()); - assertComposite(bounds.get(0), clustering_0, EOC.END); + assertEndBound(get(bounds, 0), true, clustering_0); } /** * Test 'clustering_1 = 1' with 2 clustering columns */ @Test - public void testBoundsAsCompositesWithOneEqRestrictionsAndTwoClusteringColumns() throws InvalidRequestException + public void testboundsAsClusteringWithOneEqRestrictionsAndTwoClusteringColumns() throws InvalidRequestException { CFMetaData cfMetaData = newCFMetaData(2); ByteBuffer clustering_0 = ByteBufferUtil.bytes(1); Restriction eq = newSingleEq(cfMetaData, 0, clustering_0); - PrimaryKeyRestrictions restrictions = new PrimaryKeyRestrictionSet(cfMetaData.comparator); + PrimaryKeyRestrictions restrictions = new PrimaryKeyRestrictionSet(cfMetaData.comparator, false); restrictions = restrictions.mergeWith(eq); - List bounds = restrictions.boundsAsComposites(Bound.START, QueryOptions.DEFAULT); + SortedSet bounds = restrictions.boundsAsClustering(Bound.START, QueryOptions.DEFAULT); assertEquals(1, bounds.size()); - assertComposite(bounds.get(0), clustering_0, EOC.START); + assertStartBound(get(bounds, 0), true, clustering_0); - bounds = restrictions.boundsAsComposites(Bound.END, QueryOptions.DEFAULT); + bounds = restrictions.boundsAsClustering(Bound.END, QueryOptions.DEFAULT); assertEquals(1, bounds.size()); - assertComposite(bounds.get(0), clustering_0, EOC.END); + assertEndBound(get(bounds, 0), true, clustering_0); } /** * Test 'clustering_0 IN (1, 2, 3)' with only one clustering column */ @Test - public void testBoundsAsCompositesWithOneInRestrictionsAndOneClusteringColumn() throws InvalidRequestException + public void testboundsAsClusteringWithOneInRestrictionsAndOneClusteringColumn() throws InvalidRequestException { ByteBuffer value1 = ByteBufferUtil.bytes(1); ByteBuffer value2 = ByteBufferUtil.bytes(2); @@ -120,27 +116,27 @@ public class PrimaryKeyRestrictionSetTest Restriction in = newSingleIN(cfMetaData, 0, value1, value2, value3); - PrimaryKeyRestrictions restrictions = new PrimaryKeyRestrictionSet(cfMetaData.comparator); + PrimaryKeyRestrictions restrictions = new PrimaryKeyRestrictionSet(cfMetaData.comparator, false); restrictions = restrictions.mergeWith(in); - List bounds = restrictions.boundsAsComposites(Bound.START, QueryOptions.DEFAULT); + SortedSet bounds = restrictions.boundsAsClustering(Bound.START, QueryOptions.DEFAULT); assertEquals(3, bounds.size()); - assertComposite(bounds.get(0), value1, EOC.START); - assertComposite(bounds.get(1), value2, EOC.START); - assertComposite(bounds.get(2), value3, EOC.START); + assertStartBound(get(bounds, 0), true, value1); + assertStartBound(get(bounds, 1), true, value2); + assertStartBound(get(bounds, 2), true, value3); - bounds = restrictions.boundsAsComposites(Bound.END, QueryOptions.DEFAULT); + bounds = restrictions.boundsAsClustering(Bound.END, QueryOptions.DEFAULT); assertEquals(3, bounds.size()); - assertComposite(bounds.get(0), value1, EOC.END); - assertComposite(bounds.get(1), value2, EOC.END); - assertComposite(bounds.get(2), value3, EOC.END); + assertEndBound(get(bounds, 0), true, value1); + assertEndBound(get(bounds, 1), true, value2); + assertEndBound(get(bounds, 2), true, value3); } /** * Test slice restriction (e.g 'clustering_0 > 1') with only one clustering column */ @Test - public void testBoundsAsCompositesWithSliceRestrictionsAndOneClusteringColumn() throws InvalidRequestException + public void testboundsAsClusteringWithSliceRestrictionsAndOneClusteringColumn() throws InvalidRequestException { CFMetaData cfMetaData = newCFMetaData(1); @@ -148,85 +144,85 @@ public class PrimaryKeyRestrictionSetTest ByteBuffer value2 = ByteBufferUtil.bytes(2); Restriction slice = newSingleSlice(cfMetaData, 0, Bound.START, false, value1); - PrimaryKeyRestrictions restrictions = new PrimaryKeyRestrictionSet(cfMetaData.comparator); + PrimaryKeyRestrictions restrictions = new PrimaryKeyRestrictionSet(cfMetaData.comparator, false); restrictions = restrictions.mergeWith(slice); - List bounds = restrictions.boundsAsComposites(Bound.START, QueryOptions.DEFAULT); + SortedSet bounds = restrictions.boundsAsClustering(Bound.START, QueryOptions.DEFAULT); assertEquals(1, bounds.size()); - assertComposite(bounds.get(0), value1, EOC.END); + assertStartBound(get(bounds, 0), false, value1); - bounds = restrictions.boundsAsComposites(Bound.END, QueryOptions.DEFAULT); + bounds = restrictions.boundsAsClustering(Bound.END, QueryOptions.DEFAULT); assertEquals(1, bounds.size()); - assertEmptyComposite(bounds.get(0)); + assertEmptyEnd(get(bounds, 0)); slice = newSingleSlice(cfMetaData, 0, Bound.START, true, value1); - restrictions = new PrimaryKeyRestrictionSet(cfMetaData.comparator); + restrictions = new PrimaryKeyRestrictionSet(cfMetaData.comparator, false); restrictions = restrictions.mergeWith(slice); - bounds = restrictions.boundsAsComposites(Bound.START, QueryOptions.DEFAULT); + bounds = restrictions.boundsAsClustering(Bound.START, QueryOptions.DEFAULT); assertEquals(1, bounds.size()); - assertComposite(bounds.get(0), value1, EOC.NONE); + assertStartBound(get(bounds, 0), true, value1); - bounds = restrictions.boundsAsComposites(Bound.END, QueryOptions.DEFAULT); + bounds = restrictions.boundsAsClustering(Bound.END, QueryOptions.DEFAULT); assertEquals(1, bounds.size()); - assertEmptyComposite(bounds.get(0)); + assertEmptyEnd(get(bounds, 0)); slice = newSingleSlice(cfMetaData, 0, Bound.END, true, value1); - restrictions = new PrimaryKeyRestrictionSet(cfMetaData.comparator); + restrictions = new PrimaryKeyRestrictionSet(cfMetaData.comparator, false); restrictions = restrictions.mergeWith(slice); - bounds = restrictions.boundsAsComposites(Bound.START, QueryOptions.DEFAULT); + bounds = restrictions.boundsAsClustering(Bound.START, QueryOptions.DEFAULT); assertEquals(1, bounds.size()); - assertEmptyComposite(bounds.get(0)); + assertEmptyStart(get(bounds, 0)); - bounds = restrictions.boundsAsComposites(Bound.END, QueryOptions.DEFAULT); + bounds = restrictions.boundsAsClustering(Bound.END, QueryOptions.DEFAULT); assertEquals(1, bounds.size()); - assertComposite(bounds.get(0), value1, EOC.END); + assertEndBound(get(bounds, 0), true, value1); slice = newSingleSlice(cfMetaData, 0, Bound.END, false, value1); - restrictions = new PrimaryKeyRestrictionSet(cfMetaData.comparator); + restrictions = new PrimaryKeyRestrictionSet(cfMetaData.comparator, false); restrictions = restrictions.mergeWith(slice); - bounds = restrictions.boundsAsComposites(Bound.START, QueryOptions.DEFAULT); + bounds = restrictions.boundsAsClustering(Bound.START, QueryOptions.DEFAULT); assertEquals(1, bounds.size()); - assertEmptyComposite(bounds.get(0)); + assertEmptyStart(get(bounds, 0)); - bounds = restrictions.boundsAsComposites(Bound.END, QueryOptions.DEFAULT); + bounds = restrictions.boundsAsClustering(Bound.END, QueryOptions.DEFAULT); assertEquals(1, bounds.size()); - assertComposite(bounds.get(0), value1, EOC.START); + assertEndBound(get(bounds, 0), false, value1); slice = newSingleSlice(cfMetaData, 0, Bound.START, false, value1); Restriction slice2 = newSingleSlice(cfMetaData, 0, Bound.END, false, value2); - restrictions = new PrimaryKeyRestrictionSet(cfMetaData.comparator); + restrictions = new PrimaryKeyRestrictionSet(cfMetaData.comparator, false); restrictions = restrictions.mergeWith(slice).mergeWith(slice2); - bounds = restrictions.boundsAsComposites(Bound.START, QueryOptions.DEFAULT); + bounds = restrictions.boundsAsClustering(Bound.START, QueryOptions.DEFAULT); assertEquals(1, bounds.size()); - assertComposite(bounds.get(0), value1, EOC.END); + assertStartBound(get(bounds, 0), false, value1); - bounds = restrictions.boundsAsComposites(Bound.END, QueryOptions.DEFAULT); + bounds = restrictions.boundsAsClustering(Bound.END, QueryOptions.DEFAULT); assertEquals(1, bounds.size()); - assertComposite(bounds.get(0), value2, EOC.START); + assertEndBound(get(bounds, 0), false, value2); slice = newSingleSlice(cfMetaData, 0, Bound.START, true, value1); slice2 = newSingleSlice(cfMetaData, 0, Bound.END, true, value2); - restrictions = new PrimaryKeyRestrictionSet(cfMetaData.comparator); + restrictions = new PrimaryKeyRestrictionSet(cfMetaData.comparator, false); restrictions = restrictions.mergeWith(slice).mergeWith(slice2); - bounds = restrictions.boundsAsComposites(Bound.START, QueryOptions.DEFAULT); + bounds = restrictions.boundsAsClustering(Bound.START, QueryOptions.DEFAULT); assertEquals(1, bounds.size()); - assertComposite(bounds.get(0), value1, EOC.NONE); + assertStartBound(get(bounds, 0), true, value1); - bounds = restrictions.boundsAsComposites(Bound.END, QueryOptions.DEFAULT); + bounds = restrictions.boundsAsClustering(Bound.END, QueryOptions.DEFAULT); assertEquals(1, bounds.size()); - assertComposite(bounds.get(0), value2, EOC.END); + assertEndBound(get(bounds, 0), true, value2); } /** * Test 'clustering_0 = 1 AND clustering_1 IN (1, 2, 3)' */ @Test - public void testBoundsAsCompositesWithEqAndInRestrictions() throws InvalidRequestException + public void testboundsAsClusteringWithEqAndInRestrictions() throws InvalidRequestException { CFMetaData cfMetaData = newCFMetaData(2); @@ -235,27 +231,27 @@ public class PrimaryKeyRestrictionSetTest ByteBuffer value3 = ByteBufferUtil.bytes(3); Restriction eq = newSingleEq(cfMetaData, 0, value1); Restriction in = newSingleIN(cfMetaData, 1, value1, value2, value3); - PrimaryKeyRestrictions restrictions = new PrimaryKeyRestrictionSet(cfMetaData.comparator); + PrimaryKeyRestrictions restrictions = new PrimaryKeyRestrictionSet(cfMetaData.comparator, false); restrictions = restrictions.mergeWith(eq).mergeWith(in); - List bounds = restrictions.boundsAsComposites(Bound.START, QueryOptions.DEFAULT); + SortedSet bounds = restrictions.boundsAsClustering(Bound.START, QueryOptions.DEFAULT); assertEquals(3, bounds.size()); - assertComposite(bounds.get(0), value1, value1, EOC.START); - assertComposite(bounds.get(1), value1, value2, EOC.START); - assertComposite(bounds.get(2), value1, value3, EOC.START); + assertStartBound(get(bounds, 0), true, value1, value1); + assertStartBound(get(bounds, 1), true, value1, value2); + assertStartBound(get(bounds, 2), true, value1, value3); - bounds = restrictions.boundsAsComposites(Bound.END, QueryOptions.DEFAULT); + bounds = restrictions.boundsAsClustering(Bound.END, QueryOptions.DEFAULT); assertEquals(3, bounds.size()); - assertComposite(bounds.get(0), value1, value1, EOC.END); - assertComposite(bounds.get(1), value1, value2, EOC.END); - assertComposite(bounds.get(2), value1, value3, EOC.END); + assertEndBound(get(bounds, 0), true, value1, value1); + assertEndBound(get(bounds, 1), true, value1, value2); + assertEndBound(get(bounds, 2), true, value1, value3); } /** * Test equal and slice restrictions (e.g 'clustering_0 = 0 clustering_1 > 1') */ @Test - public void testBoundsAsCompositesWithEqAndSliceRestrictions() throws InvalidRequestException + public void testboundsAsClusteringWithEqAndSliceRestrictions() throws InvalidRequestException { CFMetaData cfMetaData = newCFMetaData(2); @@ -266,108 +262,108 @@ public class PrimaryKeyRestrictionSetTest Restriction eq = newSingleEq(cfMetaData, 0, value3); Restriction slice = newSingleSlice(cfMetaData, 1, Bound.START, false, value1); - PrimaryKeyRestrictions restrictions = new PrimaryKeyRestrictionSet(cfMetaData.comparator); + PrimaryKeyRestrictions restrictions = new PrimaryKeyRestrictionSet(cfMetaData.comparator, false); restrictions = restrictions.mergeWith(eq).mergeWith(slice); - List bounds = restrictions.boundsAsComposites(Bound.START, QueryOptions.DEFAULT); + SortedSet bounds = restrictions.boundsAsClustering(Bound.START, QueryOptions.DEFAULT); assertEquals(1, bounds.size()); - assertComposite(bounds.get(0), value3, value1, EOC.END); + assertStartBound(get(bounds, 0), false, value3, value1); - bounds = restrictions.boundsAsComposites(Bound.END, QueryOptions.DEFAULT); + bounds = restrictions.boundsAsClustering(Bound.END, QueryOptions.DEFAULT); assertEquals(1, bounds.size()); - assertComposite(bounds.get(0), value3, EOC.END); + assertEndBound(get(bounds, 0), true, value3); slice = newSingleSlice(cfMetaData, 1, Bound.START, true, value1); - restrictions = new PrimaryKeyRestrictionSet(cfMetaData.comparator); + restrictions = new PrimaryKeyRestrictionSet(cfMetaData.comparator, false); restrictions = restrictions.mergeWith(eq).mergeWith(slice); - bounds = restrictions.boundsAsComposites(Bound.START, QueryOptions.DEFAULT); + bounds = restrictions.boundsAsClustering(Bound.START, QueryOptions.DEFAULT); assertEquals(1, bounds.size()); - assertComposite(bounds.get(0), value3, value1, EOC.NONE); + assertStartBound(get(bounds, 0), true, value3, value1); - bounds = restrictions.boundsAsComposites(Bound.END, QueryOptions.DEFAULT); + bounds = restrictions.boundsAsClustering(Bound.END, QueryOptions.DEFAULT); assertEquals(1, bounds.size()); - assertComposite(bounds.get(0), value3, EOC.END); + assertEndBound(get(bounds, 0), true, value3); slice = newSingleSlice(cfMetaData, 1, Bound.END, true, value1); - restrictions = new PrimaryKeyRestrictionSet(cfMetaData.comparator); + restrictions = new PrimaryKeyRestrictionSet(cfMetaData.comparator, false); restrictions = restrictions.mergeWith(eq).mergeWith(slice); - bounds = restrictions.boundsAsComposites(Bound.START, QueryOptions.DEFAULT); + bounds = restrictions.boundsAsClustering(Bound.START, QueryOptions.DEFAULT); assertEquals(1, bounds.size()); - assertComposite(bounds.get(0), value3, EOC.START); + assertStartBound(get(bounds, 0), true, value3); - bounds = restrictions.boundsAsComposites(Bound.END, QueryOptions.DEFAULT); + bounds = restrictions.boundsAsClustering(Bound.END, QueryOptions.DEFAULT); assertEquals(1, bounds.size()); - assertComposite(bounds.get(0), value3, value1, EOC.END); + assertEndBound(get(bounds, 0), true, value3, value1); slice = newSingleSlice(cfMetaData, 1, Bound.END, false, value1); - restrictions = new PrimaryKeyRestrictionSet(cfMetaData.comparator); + restrictions = new PrimaryKeyRestrictionSet(cfMetaData.comparator, false); restrictions = restrictions.mergeWith(eq).mergeWith(slice); - bounds = restrictions.boundsAsComposites(Bound.START, QueryOptions.DEFAULT); + bounds = restrictions.boundsAsClustering(Bound.START, QueryOptions.DEFAULT); assertEquals(1, bounds.size()); - assertComposite(bounds.get(0), value3, EOC.START); + assertStartBound(get(bounds, 0), true, value3); - bounds = restrictions.boundsAsComposites(Bound.END, QueryOptions.DEFAULT); + bounds = restrictions.boundsAsClustering(Bound.END, QueryOptions.DEFAULT); assertEquals(1, bounds.size()); - assertComposite(bounds.get(0), value3, value1, EOC.START); + assertEndBound(get(bounds, 0), false, value3, value1); slice = newSingleSlice(cfMetaData, 1, Bound.START, false, value1); Restriction slice2 = newSingleSlice(cfMetaData, 1, Bound.END, false, value2); - restrictions = new PrimaryKeyRestrictionSet(cfMetaData.comparator); + restrictions = new PrimaryKeyRestrictionSet(cfMetaData.comparator, false); restrictions = restrictions.mergeWith(eq).mergeWith(slice).mergeWith(slice2); - bounds = restrictions.boundsAsComposites(Bound.START, QueryOptions.DEFAULT); + bounds = restrictions.boundsAsClustering(Bound.START, QueryOptions.DEFAULT); assertEquals(1, bounds.size()); - assertComposite(bounds.get(0), value3, value1, EOC.END); + assertStartBound(get(bounds, 0), false, value3, value1); - bounds = restrictions.boundsAsComposites(Bound.END, QueryOptions.DEFAULT); + bounds = restrictions.boundsAsClustering(Bound.END, QueryOptions.DEFAULT); assertEquals(1, bounds.size()); - assertComposite(bounds.get(0), value3, value2, EOC.START); + assertEndBound(get(bounds, 0), false, value3, value2); slice = newSingleSlice(cfMetaData, 1, Bound.START, true, value1); slice2 = newSingleSlice(cfMetaData, 1, Bound.END, true, value2); - restrictions = new PrimaryKeyRestrictionSet(cfMetaData.comparator); + restrictions = new PrimaryKeyRestrictionSet(cfMetaData.comparator, false); restrictions = restrictions.mergeWith(eq).mergeWith(slice).mergeWith(slice2); - bounds = restrictions.boundsAsComposites(Bound.START, QueryOptions.DEFAULT); + bounds = restrictions.boundsAsClustering(Bound.START, QueryOptions.DEFAULT); assertEquals(1, bounds.size()); - assertComposite(bounds.get(0), value3, value1, EOC.NONE); + assertStartBound(get(bounds, 0), true, value3, value1); - bounds = restrictions.boundsAsComposites(Bound.END, QueryOptions.DEFAULT); + bounds = restrictions.boundsAsClustering(Bound.END, QueryOptions.DEFAULT); assertEquals(1, bounds.size()); - assertComposite(bounds.get(0), value3, value2, EOC.END); + assertEndBound(get(bounds, 0), true, value3, value2); } /** * Test '(clustering_0, clustering_1) = (1, 2)' with two clustering column */ @Test - public void testBoundsAsCompositesWithMultiEqRestrictions() throws InvalidRequestException + public void testboundsAsClusteringWithMultiEqRestrictions() throws InvalidRequestException { CFMetaData cfMetaData = newCFMetaData(2); ByteBuffer value1 = ByteBufferUtil.bytes(1); ByteBuffer value2 = ByteBufferUtil.bytes(2); Restriction eq = newMultiEq(cfMetaData, 0, value1, value2); - PrimaryKeyRestrictions restrictions = new PrimaryKeyRestrictionSet(cfMetaData.comparator); + PrimaryKeyRestrictions restrictions = new PrimaryKeyRestrictionSet(cfMetaData.comparator, false); restrictions = restrictions.mergeWith(eq); - List bounds = restrictions.boundsAsComposites(Bound.START, QueryOptions.DEFAULT); + SortedSet bounds = restrictions.boundsAsClustering(Bound.START, QueryOptions.DEFAULT); assertEquals(1, bounds.size()); - assertComposite(bounds.get(0), value1, value2, EOC.START); + assertStartBound(get(bounds, 0), true, value1, value2); - bounds = restrictions.boundsAsComposites(Bound.END, QueryOptions.DEFAULT); + bounds = restrictions.boundsAsClustering(Bound.END, QueryOptions.DEFAULT); assertEquals(1, bounds.size()); - assertComposite(bounds.get(0), value1, value2, EOC.END); + assertEndBound(get(bounds, 0), true, value1, value2); } /** * Test '(clustering_0, clustering_1) IN ((1, 2), (2, 3))' with two clustering column */ @Test - public void testBoundsAsCompositesWithMultiInRestrictions() throws InvalidRequestException + public void testboundsAsClusteringWithMultiInRestrictions() throws InvalidRequestException { CFMetaData cfMetaData = newCFMetaData(2); @@ -375,25 +371,25 @@ public class PrimaryKeyRestrictionSetTest ByteBuffer value2 = ByteBufferUtil.bytes(2); ByteBuffer value3 = ByteBufferUtil.bytes(3); Restriction in = newMultiIN(cfMetaData, 0, asList(value1, value2), asList(value2, value3)); - PrimaryKeyRestrictions restrictions = new PrimaryKeyRestrictionSet(cfMetaData.comparator); + PrimaryKeyRestrictions restrictions = new PrimaryKeyRestrictionSet(cfMetaData.comparator, false); restrictions = restrictions.mergeWith(in); - List bounds = restrictions.boundsAsComposites(Bound.START, QueryOptions.DEFAULT); + SortedSet bounds = restrictions.boundsAsClustering(Bound.START, QueryOptions.DEFAULT); assertEquals(2, bounds.size()); - assertComposite(bounds.get(0), value1, value2, EOC.START); - assertComposite(bounds.get(1), value2, value3, EOC.START); + assertStartBound(get(bounds, 0), true, value1, value2); + assertStartBound(get(bounds, 1), true, value2, value3); - bounds = restrictions.boundsAsComposites(Bound.END, QueryOptions.DEFAULT); + bounds = restrictions.boundsAsClustering(Bound.END, QueryOptions.DEFAULT); assertEquals(2, bounds.size()); - assertComposite(bounds.get(0), value1, value2, EOC.END); - assertComposite(bounds.get(1), value2, value3, EOC.END); + assertEndBound(get(bounds, 0), true, value1, value2); + assertEndBound(get(bounds, 1), true, value2, value3); } /** * Test multi-column slice restrictions (e.g '(clustering_0) > (1)') with only one clustering column */ @Test - public void testBoundsAsCompositesWithMultiSliceRestrictionsWithOneClusteringColumn() throws InvalidRequestException + public void testboundsAsClusteringWithMultiSliceRestrictionsWithOneClusteringColumn() throws InvalidRequestException { CFMetaData cfMetaData = newCFMetaData(1); @@ -401,85 +397,85 @@ public class PrimaryKeyRestrictionSetTest ByteBuffer value2 = ByteBufferUtil.bytes(2); Restriction slice = newMultiSlice(cfMetaData, 0, Bound.START, false, value1); - PrimaryKeyRestrictions restrictions = new PrimaryKeyRestrictionSet(cfMetaData.comparator); + PrimaryKeyRestrictions restrictions = new PrimaryKeyRestrictionSet(cfMetaData.comparator, false); restrictions = restrictions.mergeWith(slice); - List bounds = restrictions.boundsAsComposites(Bound.START, QueryOptions.DEFAULT); + SortedSet bounds = restrictions.boundsAsClustering(Bound.START, QueryOptions.DEFAULT); assertEquals(1, bounds.size()); - assertComposite(bounds.get(0), value1, EOC.END); + assertStartBound(get(bounds, 0), false, value1); - bounds = restrictions.boundsAsComposites(Bound.END, QueryOptions.DEFAULT); + bounds = restrictions.boundsAsClustering(Bound.END, QueryOptions.DEFAULT); assertEquals(1, bounds.size()); - assertEmptyComposite(bounds.get(0)); + assertEmptyEnd(get(bounds, 0)); slice = newMultiSlice(cfMetaData, 0, Bound.START, true, value1); - restrictions = new PrimaryKeyRestrictionSet(cfMetaData.comparator); + restrictions = new PrimaryKeyRestrictionSet(cfMetaData.comparator, false); restrictions = restrictions.mergeWith(slice); - bounds = restrictions.boundsAsComposites(Bound.START, QueryOptions.DEFAULT); + bounds = restrictions.boundsAsClustering(Bound.START, QueryOptions.DEFAULT); assertEquals(1, bounds.size()); - assertComposite(bounds.get(0), value1, EOC.NONE); + assertStartBound(get(bounds, 0), true, value1); - bounds = restrictions.boundsAsComposites(Bound.END, QueryOptions.DEFAULT); + bounds = restrictions.boundsAsClustering(Bound.END, QueryOptions.DEFAULT); assertEquals(1, bounds.size()); - assertEmptyComposite(bounds.get(0)); + assertEmptyEnd(get(bounds, 0)); slice = newMultiSlice(cfMetaData, 0, Bound.END, true, value1); - restrictions = new PrimaryKeyRestrictionSet(cfMetaData.comparator); + restrictions = new PrimaryKeyRestrictionSet(cfMetaData.comparator, false); restrictions = restrictions.mergeWith(slice); - bounds = restrictions.boundsAsComposites(Bound.START, QueryOptions.DEFAULT); + bounds = restrictions.boundsAsClustering(Bound.START, QueryOptions.DEFAULT); assertEquals(1, bounds.size()); - assertEmptyComposite(bounds.get(0)); + assertEmptyStart(get(bounds, 0)); - bounds = restrictions.boundsAsComposites(Bound.END, QueryOptions.DEFAULT); + bounds = restrictions.boundsAsClustering(Bound.END, QueryOptions.DEFAULT); assertEquals(1, bounds.size()); - assertComposite(bounds.get(0), value1, EOC.END); + assertEndBound(get(bounds, 0), true, value1); slice = newMultiSlice(cfMetaData, 0, Bound.END, false, value1); - restrictions = new PrimaryKeyRestrictionSet(cfMetaData.comparator); + restrictions = new PrimaryKeyRestrictionSet(cfMetaData.comparator, false); restrictions = restrictions.mergeWith(slice); - bounds = restrictions.boundsAsComposites(Bound.START, QueryOptions.DEFAULT); + bounds = restrictions.boundsAsClustering(Bound.START, QueryOptions.DEFAULT); assertEquals(1, bounds.size()); - assertEmptyComposite(bounds.get(0)); + assertEmptyStart(get(bounds, 0)); - bounds = restrictions.boundsAsComposites(Bound.END, QueryOptions.DEFAULT); + bounds = restrictions.boundsAsClustering(Bound.END, QueryOptions.DEFAULT); assertEquals(1, bounds.size()); - assertComposite(bounds.get(0), value1, EOC.START); + assertEndBound(get(bounds, 0), false, value1); slice = newMultiSlice(cfMetaData, 0, Bound.START, false, value1); Restriction slice2 = newMultiSlice(cfMetaData, 0, Bound.END, false, value2); - restrictions = new PrimaryKeyRestrictionSet(cfMetaData.comparator); + restrictions = new PrimaryKeyRestrictionSet(cfMetaData.comparator, false); restrictions = restrictions.mergeWith(slice).mergeWith(slice2); - bounds = restrictions.boundsAsComposites(Bound.START, QueryOptions.DEFAULT); + bounds = restrictions.boundsAsClustering(Bound.START, QueryOptions.DEFAULT); assertEquals(1, bounds.size()); - assertComposite(bounds.get(0), value1, EOC.END); + assertStartBound(get(bounds, 0), false, value1); - bounds = restrictions.boundsAsComposites(Bound.END, QueryOptions.DEFAULT); + bounds = restrictions.boundsAsClustering(Bound.END, QueryOptions.DEFAULT); assertEquals(1, bounds.size()); - assertComposite(bounds.get(0), value2, EOC.START); + assertEndBound(get(bounds, 0), false, value2); slice = newMultiSlice(cfMetaData, 0, Bound.START, true, value1); slice2 = newMultiSlice(cfMetaData, 0, Bound.END, true, value2); - restrictions = new PrimaryKeyRestrictionSet(cfMetaData.comparator); + restrictions = new PrimaryKeyRestrictionSet(cfMetaData.comparator, false); restrictions = restrictions.mergeWith(slice).mergeWith(slice2); - bounds = restrictions.boundsAsComposites(Bound.START, QueryOptions.DEFAULT); + bounds = restrictions.boundsAsClustering(Bound.START, QueryOptions.DEFAULT); assertEquals(1, bounds.size()); - assertComposite(bounds.get(0), value1, EOC.NONE); + assertStartBound(get(bounds, 0), true, value1); - bounds = restrictions.boundsAsComposites(Bound.END, QueryOptions.DEFAULT); + bounds = restrictions.boundsAsClustering(Bound.END, QueryOptions.DEFAULT); assertEquals(1, bounds.size()); - assertComposite(bounds.get(0), value2, EOC.END); + assertEndBound(get(bounds, 0), true, value2); } /** * Test multi-column slice restrictions (e.g '(clustering_0, clustering_1) > (1, 2)') */ @Test - public void testBoundsAsCompositesWithMultiSliceRestrictionsWithTwoClusteringColumn() throws InvalidRequestException + public void testboundsAsClusteringWithMultiSliceRestrictionsWithTwoClusteringColumn() throws InvalidRequestException { CFMetaData cfMetaData = newCFMetaData(2); @@ -488,90 +484,90 @@ public class PrimaryKeyRestrictionSetTest // (clustering_0, clustering1) > (1, 2) Restriction slice = newMultiSlice(cfMetaData, 0, Bound.START, false, value1, value2); - PrimaryKeyRestrictions restrictions = new PrimaryKeyRestrictionSet(cfMetaData.comparator); + PrimaryKeyRestrictions restrictions = new PrimaryKeyRestrictionSet(cfMetaData.comparator, false); restrictions = restrictions.mergeWith(slice); - List bounds = restrictions.boundsAsComposites(Bound.START, QueryOptions.DEFAULT); + SortedSet bounds = restrictions.boundsAsClustering(Bound.START, QueryOptions.DEFAULT); assertEquals(1, bounds.size()); - assertComposite(bounds.get(0), value1, value2, EOC.END); + assertStartBound(get(bounds, 0), false, value1, value2); - bounds = restrictions.boundsAsComposites(Bound.END, QueryOptions.DEFAULT); + bounds = restrictions.boundsAsClustering(Bound.END, QueryOptions.DEFAULT); assertEquals(1, bounds.size()); - assertEmptyComposite(bounds.get(0)); + assertEmptyEnd(get(bounds, 0)); // (clustering_0, clustering1) >= (1, 2) slice = newMultiSlice(cfMetaData, 0, Bound.START, true, value1, value2); - restrictions = new PrimaryKeyRestrictionSet(cfMetaData.comparator); + restrictions = new PrimaryKeyRestrictionSet(cfMetaData.comparator, false); restrictions = restrictions.mergeWith(slice); - bounds = restrictions.boundsAsComposites(Bound.START, QueryOptions.DEFAULT); + bounds = restrictions.boundsAsClustering(Bound.START, QueryOptions.DEFAULT); assertEquals(1, bounds.size()); - assertComposite(bounds.get(0), value1, value2, EOC.NONE); + assertStartBound(get(bounds, 0), true, value1, value2); - bounds = restrictions.boundsAsComposites(Bound.END, QueryOptions.DEFAULT); + bounds = restrictions.boundsAsClustering(Bound.END, QueryOptions.DEFAULT); assertEquals(1, bounds.size()); - assertEmptyComposite(bounds.get(0)); + assertEmptyEnd(get(bounds, 0)); // (clustering_0, clustering1) <= (1, 2) slice = newMultiSlice(cfMetaData, 0, Bound.END, true, value1, value2); - restrictions = new PrimaryKeyRestrictionSet(cfMetaData.comparator); + restrictions = new PrimaryKeyRestrictionSet(cfMetaData.comparator, false); restrictions = restrictions.mergeWith(slice); - bounds = restrictions.boundsAsComposites(Bound.START, QueryOptions.DEFAULT); + bounds = restrictions.boundsAsClustering(Bound.START, QueryOptions.DEFAULT); assertEquals(1, bounds.size()); - assertEmptyComposite(bounds.get(0)); + assertEmptyStart(get(bounds, 0)); - bounds = restrictions.boundsAsComposites(Bound.END, QueryOptions.DEFAULT); + bounds = restrictions.boundsAsClustering(Bound.END, QueryOptions.DEFAULT); assertEquals(1, bounds.size()); - assertComposite(bounds.get(0), value1, value2, EOC.END); + assertEndBound(get(bounds, 0), true, value1, value2); // (clustering_0, clustering1) < (1, 2) slice = newMultiSlice(cfMetaData, 0, Bound.END, false, value1, value2); - restrictions = new PrimaryKeyRestrictionSet(cfMetaData.comparator); + restrictions = new PrimaryKeyRestrictionSet(cfMetaData.comparator, false); restrictions = restrictions.mergeWith(slice); - bounds = restrictions.boundsAsComposites(Bound.START, QueryOptions.DEFAULT); + bounds = restrictions.boundsAsClustering(Bound.START, QueryOptions.DEFAULT); assertEquals(1, bounds.size()); - assertEmptyComposite(bounds.get(0)); + assertEmptyStart(get(bounds, 0)); - bounds = restrictions.boundsAsComposites(Bound.END, QueryOptions.DEFAULT); + bounds = restrictions.boundsAsClustering(Bound.END, QueryOptions.DEFAULT); assertEquals(1, bounds.size()); - assertComposite(bounds.get(0), value1, value2, EOC.START); + assertEndBound(get(bounds, 0), false, value1, value2); // (clustering_0, clustering1) > (1, 2) AND (clustering_0) < (2) slice = newMultiSlice(cfMetaData, 0, Bound.START, false, value1, value2); Restriction slice2 = newMultiSlice(cfMetaData, 0, Bound.END, false, value2); - restrictions = new PrimaryKeyRestrictionSet(cfMetaData.comparator); + restrictions = new PrimaryKeyRestrictionSet(cfMetaData.comparator, false); restrictions = restrictions.mergeWith(slice).mergeWith(slice2); - bounds = restrictions.boundsAsComposites(Bound.START, QueryOptions.DEFAULT); + bounds = restrictions.boundsAsClustering(Bound.START, QueryOptions.DEFAULT); assertEquals(1, bounds.size()); - assertComposite(bounds.get(0), value1, value2, EOC.END); + assertStartBound(get(bounds, 0), false, value1, value2); - bounds = restrictions.boundsAsComposites(Bound.END, QueryOptions.DEFAULT); + bounds = restrictions.boundsAsClustering(Bound.END, QueryOptions.DEFAULT); assertEquals(1, bounds.size()); - assertComposite(bounds.get(0), value2, EOC.START); + assertEndBound(get(bounds, 0), false, value2); // (clustering_0, clustering1) >= (1, 2) AND (clustering_0, clustering1) <= (2, 1) slice = newMultiSlice(cfMetaData, 0, Bound.START, true, value1, value2); slice2 = newMultiSlice(cfMetaData, 0, Bound.END, true, value2, value1); - restrictions = new PrimaryKeyRestrictionSet(cfMetaData.comparator); + restrictions = new PrimaryKeyRestrictionSet(cfMetaData.comparator, false); restrictions = restrictions.mergeWith(slice).mergeWith(slice2); - bounds = restrictions.boundsAsComposites(Bound.START, QueryOptions.DEFAULT); + bounds = restrictions.boundsAsClustering(Bound.START, QueryOptions.DEFAULT); assertEquals(1, bounds.size()); - assertComposite(bounds.get(0), value1, value2, EOC.NONE); + assertStartBound(get(bounds, 0), true, value1, value2); - bounds = restrictions.boundsAsComposites(Bound.END, QueryOptions.DEFAULT); + bounds = restrictions.boundsAsClustering(Bound.END, QueryOptions.DEFAULT); assertEquals(1, bounds.size()); - assertComposite(bounds.get(0), value2, value1, EOC.END); + assertEndBound(get(bounds, 0), true, value2, value1); } /** * Test mixing single and multi equals restrictions (e.g. clustering_0 = 1 AND (clustering_1, clustering_2) = (2, 3)) */ @Test - public void testBoundsAsCompositesWithSingleEqAndMultiEqRestrictions() throws InvalidRequestException + public void testboundsAsClusteringWithSingleEqAndMultiEqRestrictions() throws InvalidRequestException { CFMetaData cfMetaData = newCFMetaData(4); @@ -583,67 +579,67 @@ public class PrimaryKeyRestrictionSetTest // clustering_0 = 1 AND (clustering_1, clustering_2) = (2, 3) Restriction singleEq = newSingleEq(cfMetaData, 0, value1); Restriction multiEq = newMultiEq(cfMetaData, 1, value2, value3); - PrimaryKeyRestrictions restrictions = new PrimaryKeyRestrictionSet(cfMetaData.comparator); + PrimaryKeyRestrictions restrictions = new PrimaryKeyRestrictionSet(cfMetaData.comparator, false); restrictions = restrictions.mergeWith(singleEq).mergeWith(multiEq); - List bounds = restrictions.boundsAsComposites(Bound.START, QueryOptions.DEFAULT); + SortedSet bounds = restrictions.boundsAsClustering(Bound.START, QueryOptions.DEFAULT); assertEquals(1, bounds.size()); - assertComposite(bounds.get(0), value1, value2, value3, EOC.START); + assertStartBound(get(bounds, 0), true, value1, value2, value3); - bounds = restrictions.boundsAsComposites(Bound.END, QueryOptions.DEFAULT); + bounds = restrictions.boundsAsClustering(Bound.END, QueryOptions.DEFAULT); assertEquals(1, bounds.size()); - assertComposite(bounds.get(0), value1, value2, value3, EOC.END); + assertEndBound(get(bounds, 0), true, value1, value2, value3); // clustering_0 = 1 AND clustering_1 = 2 AND (clustering_2, clustering_3) = (3, 4) singleEq = newSingleEq(cfMetaData, 0, value1); Restriction singleEq2 = newSingleEq(cfMetaData, 1, value2); multiEq = newMultiEq(cfMetaData, 2, value3, value4); - restrictions = new PrimaryKeyRestrictionSet(cfMetaData.comparator); + restrictions = new PrimaryKeyRestrictionSet(cfMetaData.comparator, false); restrictions = restrictions.mergeWith(singleEq).mergeWith(singleEq2).mergeWith(multiEq); - bounds = restrictions.boundsAsComposites(Bound.START, QueryOptions.DEFAULT); + bounds = restrictions.boundsAsClustering(Bound.START, QueryOptions.DEFAULT); assertEquals(1, bounds.size()); - assertComposite(bounds.get(0), value1, value2, value3, value4, EOC.START); + assertStartBound(get(bounds, 0), true, value1, value2, value3, value4); - bounds = restrictions.boundsAsComposites(Bound.END, QueryOptions.DEFAULT); + bounds = restrictions.boundsAsClustering(Bound.END, QueryOptions.DEFAULT); assertEquals(1, bounds.size()); - assertComposite(bounds.get(0), value1, value2, value3, value4, EOC.END); + assertEndBound(get(bounds, 0), true, value1, value2, value3, value4); // (clustering_0, clustering_1) = (1, 2) AND clustering_2 = 3 singleEq = newSingleEq(cfMetaData, 2, value3); multiEq = newMultiEq(cfMetaData, 0, value1, value2); - restrictions = new PrimaryKeyRestrictionSet(cfMetaData.comparator); + restrictions = new PrimaryKeyRestrictionSet(cfMetaData.comparator, false); restrictions = restrictions.mergeWith(singleEq).mergeWith(multiEq); - bounds = restrictions.boundsAsComposites(Bound.START, QueryOptions.DEFAULT); + bounds = restrictions.boundsAsClustering(Bound.START, QueryOptions.DEFAULT); assertEquals(1, bounds.size()); - assertComposite(bounds.get(0), value1, value2, value3, EOC.START); + assertStartBound(get(bounds, 0), true, value1, value2, value3); - bounds = restrictions.boundsAsComposites(Bound.END, QueryOptions.DEFAULT); + bounds = restrictions.boundsAsClustering(Bound.END, QueryOptions.DEFAULT); assertEquals(1, bounds.size()); - assertComposite(bounds.get(0), value1, value2, value3, EOC.END); + assertEndBound(get(bounds, 0), true, value1, value2, value3); // clustering_0 = 1 AND (clustering_1, clustering_2) = (2, 3) AND clustering_3 = 4 singleEq = newSingleEq(cfMetaData, 0, value1); singleEq2 = newSingleEq(cfMetaData, 3, value4); multiEq = newMultiEq(cfMetaData, 1, value2, value3); - restrictions = new PrimaryKeyRestrictionSet(cfMetaData.comparator); + restrictions = new PrimaryKeyRestrictionSet(cfMetaData.comparator, false); restrictions = restrictions.mergeWith(singleEq).mergeWith(multiEq).mergeWith(singleEq2); - bounds = restrictions.boundsAsComposites(Bound.START, QueryOptions.DEFAULT); + bounds = restrictions.boundsAsClustering(Bound.START, QueryOptions.DEFAULT); assertEquals(1, bounds.size()); - assertComposite(bounds.get(0), value1, value2, value3, value4, EOC.START); + assertStartBound(get(bounds, 0), true, value1, value2, value3, value4); - bounds = restrictions.boundsAsComposites(Bound.END, QueryOptions.DEFAULT); + bounds = restrictions.boundsAsClustering(Bound.END, QueryOptions.DEFAULT); assertEquals(1, bounds.size()); - assertComposite(bounds.get(0), value1, value2, value3, value4, EOC.END); + assertEndBound(get(bounds, 0), true, value1, value2, value3, value4); } /** * Test clustering_0 = 1 AND (clustering_1, clustering_2) IN ((2, 3), (4, 5)) */ @Test - public void testBoundsAsCompositesWithSingleEqAndMultiINRestrictions() throws InvalidRequestException + public void testboundsAsClusteringWithSingleEqAndMultiINRestrictions() throws InvalidRequestException { CFMetaData cfMetaData = newCFMetaData(4); @@ -656,49 +652,49 @@ public class PrimaryKeyRestrictionSetTest // clustering_0 = 1 AND (clustering_1, clustering_2) IN ((2, 3), (4, 5)) Restriction singleEq = newSingleEq(cfMetaData, 0, value1); Restriction multiIN = newMultiIN(cfMetaData, 1, asList(value2, value3), asList(value4, value5)); - PrimaryKeyRestrictions restrictions = new PrimaryKeyRestrictionSet(cfMetaData.comparator); + PrimaryKeyRestrictions restrictions = new PrimaryKeyRestrictionSet(cfMetaData.comparator, false); restrictions = restrictions.mergeWith(singleEq).mergeWith(multiIN); - List bounds = restrictions.boundsAsComposites(Bound.START, QueryOptions.DEFAULT); + SortedSet bounds = restrictions.boundsAsClustering(Bound.START, QueryOptions.DEFAULT); assertEquals(2, bounds.size()); - assertComposite(bounds.get(0), value1, value2, value3, EOC.START); - assertComposite(bounds.get(1), value1, value4, value5, EOC.START); + assertStartBound(get(bounds, 0), true, value1, value2, value3); + assertStartBound(get(bounds, 1), true, value1, value4, value5); - bounds = restrictions.boundsAsComposites(Bound.END, QueryOptions.DEFAULT); + bounds = restrictions.boundsAsClustering(Bound.END, QueryOptions.DEFAULT); assertEquals(2, bounds.size()); - assertComposite(bounds.get(0), value1, value2, value3, EOC.END); - assertComposite(bounds.get(1), value1, value4, value5, EOC.END); + assertEndBound(get(bounds, 0), true, value1, value2, value3); + assertEndBound(get(bounds, 1), true, value1, value4, value5); // clustering_0 = 1 AND (clustering_1, clustering_2) IN ((2, 3)) singleEq = newSingleEq(cfMetaData, 0, value1); multiIN = newMultiIN(cfMetaData, 1, asList(value2, value3)); - restrictions = new PrimaryKeyRestrictionSet(cfMetaData.comparator); + restrictions = new PrimaryKeyRestrictionSet(cfMetaData.comparator, false); restrictions = restrictions.mergeWith(multiIN).mergeWith(singleEq); - bounds = restrictions.boundsAsComposites(Bound.START, QueryOptions.DEFAULT); + bounds = restrictions.boundsAsClustering(Bound.START, QueryOptions.DEFAULT); assertEquals(1, bounds.size()); - assertComposite(bounds.get(0), value1, value2, value3, EOC.START); + assertStartBound(get(bounds, 0), true, value1, value2, value3); - bounds = restrictions.boundsAsComposites(Bound.END, QueryOptions.DEFAULT); + bounds = restrictions.boundsAsClustering(Bound.END, QueryOptions.DEFAULT); assertEquals(1, bounds.size()); - assertComposite(bounds.get(0), value1, value2, value3, EOC.END); + assertEndBound(get(bounds, 0), true, value1, value2, value3); // clustering_0 = 1 AND clustering_1 = 5 AND (clustering_2, clustering_3) IN ((2, 3), (4, 5)) singleEq = newSingleEq(cfMetaData, 0, value1); Restriction singleEq2 = newSingleEq(cfMetaData, 1, value5); multiIN = newMultiIN(cfMetaData, 2, asList(value2, value3), asList(value4, value5)); - restrictions = new PrimaryKeyRestrictionSet(cfMetaData.comparator); + restrictions = new PrimaryKeyRestrictionSet(cfMetaData.comparator, false); restrictions = restrictions.mergeWith(singleEq).mergeWith(multiIN).mergeWith(singleEq2); - bounds = restrictions.boundsAsComposites(Bound.START, QueryOptions.DEFAULT); + bounds = restrictions.boundsAsClustering(Bound.START, QueryOptions.DEFAULT); assertEquals(2, bounds.size()); - assertComposite(bounds.get(0), value1, value5, value2, value3, EOC.START); - assertComposite(bounds.get(1), value1, value5, value4, value5, EOC.START); + assertStartBound(get(bounds, 0), true, value1, value5, value2, value3); + assertStartBound(get(bounds, 1), true, value1, value5, value4, value5); - bounds = restrictions.boundsAsComposites(Bound.END, QueryOptions.DEFAULT); + bounds = restrictions.boundsAsClustering(Bound.END, QueryOptions.DEFAULT); assertEquals(2, bounds.size()); - assertComposite(bounds.get(0), value1, value5, value2, value3, EOC.END); - assertComposite(bounds.get(1), value1, value5, value4, value5, EOC.END); + assertEndBound(get(bounds, 0), true, value1, value5, value2, value3); + assertEndBound(get(bounds, 1), true, value1, value5, value4, value5); } /** @@ -706,7 +702,7 @@ public class PrimaryKeyRestrictionSetTest * (e.g. clustering_0 = 1 AND (clustering_1, clustering_2) > (2, 3)) */ @Test - public void testBoundsAsCompositesWithSingleEqAndSliceRestrictions() throws InvalidRequestException + public void testboundsAsClusteringWithSingleEqAndSliceRestrictions() throws InvalidRequestException { CFMetaData cfMetaData = newCFMetaData(3); @@ -719,46 +715,46 @@ public class PrimaryKeyRestrictionSetTest // clustering_0 = 1 AND (clustering_1, clustering_2) > (2, 3) Restriction singleEq = newSingleEq(cfMetaData, 0, value1); Restriction multiSlice = newMultiSlice(cfMetaData, 1, Bound.START, false, value2, value3); - PrimaryKeyRestrictions restrictions = new PrimaryKeyRestrictionSet(cfMetaData.comparator); + PrimaryKeyRestrictions restrictions = new PrimaryKeyRestrictionSet(cfMetaData.comparator, false); restrictions = restrictions.mergeWith(singleEq).mergeWith(multiSlice); - List bounds = restrictions.boundsAsComposites(Bound.START, QueryOptions.DEFAULT); + SortedSet bounds = restrictions.boundsAsClustering(Bound.START, QueryOptions.DEFAULT); assertEquals(1, bounds.size()); - assertComposite(bounds.get(0), value1, value2, value3, EOC.END); + assertStartBound(get(bounds, 0), false, value1, value2, value3); - bounds = restrictions.boundsAsComposites(Bound.END, QueryOptions.DEFAULT); + bounds = restrictions.boundsAsClustering(Bound.END, QueryOptions.DEFAULT); assertEquals(1, bounds.size()); - assertComposite(bounds.get(0), value1, EOC.END); + assertEndBound(get(bounds, 0), true, value1); // clustering_0 = 1 AND (clustering_1, clustering_2) > (2, 3) AND (clustering_1) < (4) singleEq = newSingleEq(cfMetaData, 0, value1); multiSlice = newMultiSlice(cfMetaData, 1, Bound.START, false, value2, value3); Restriction multiSlice2 = newMultiSlice(cfMetaData, 1, Bound.END, false, value4); - restrictions = new PrimaryKeyRestrictionSet(cfMetaData.comparator); + restrictions = new PrimaryKeyRestrictionSet(cfMetaData.comparator, false); restrictions = restrictions.mergeWith(multiSlice2).mergeWith(singleEq).mergeWith(multiSlice); - bounds = restrictions.boundsAsComposites(Bound.START, QueryOptions.DEFAULT); + bounds = restrictions.boundsAsClustering(Bound.START, QueryOptions.DEFAULT); assertEquals(1, bounds.size()); - assertComposite(bounds.get(0), value1, value2, value3, EOC.END); + assertStartBound(get(bounds, 0), false, value1, value2, value3); - bounds = restrictions.boundsAsComposites(Bound.END, QueryOptions.DEFAULT); + bounds = restrictions.boundsAsClustering(Bound.END, QueryOptions.DEFAULT); assertEquals(1, bounds.size()); - assertComposite(bounds.get(0), value1, value4, EOC.START); + assertEndBound(get(bounds, 0), false, value1, value4); // clustering_0 = 1 AND (clustering_1, clustering_2) => (2, 3) AND (clustering_1, clustering_2) <= (4, 5) singleEq = newSingleEq(cfMetaData, 0, value1); multiSlice = newMultiSlice(cfMetaData, 1, Bound.START, true, value2, value3); multiSlice2 = newMultiSlice(cfMetaData, 1, Bound.END, true, value4, value5); - restrictions = new PrimaryKeyRestrictionSet(cfMetaData.comparator); + restrictions = new PrimaryKeyRestrictionSet(cfMetaData.comparator, false); restrictions = restrictions.mergeWith(multiSlice2).mergeWith(singleEq).mergeWith(multiSlice); - bounds = restrictions.boundsAsComposites(Bound.START, QueryOptions.DEFAULT); + bounds = restrictions.boundsAsClustering(Bound.START, QueryOptions.DEFAULT); assertEquals(1, bounds.size()); - assertComposite(bounds.get(0), value1, value2, value3, EOC.NONE); + assertStartBound(get(bounds, 0), true, value1, value2, value3); - bounds = restrictions.boundsAsComposites(Bound.END, QueryOptions.DEFAULT); + bounds = restrictions.boundsAsClustering(Bound.END, QueryOptions.DEFAULT); assertEquals(1, bounds.size()); - assertComposite(bounds.get(0), value1, value4, value5, EOC.END); + assertEndBound(get(bounds, 0), true, value1, value4, value5); } /** @@ -766,7 +762,7 @@ public class PrimaryKeyRestrictionSetTest * (e.g. clustering_0 = 1 AND (clustering_1, clustering_2) > (2, 3)) */ @Test - public void testBoundsAsCompositesWithMultiEqAndSingleSliceRestrictions() throws InvalidRequestException + public void testboundsAsClusteringWithMultiEqAndSingleSliceRestrictions() throws InvalidRequestException { CFMetaData cfMetaData = newCFMetaData(3); @@ -777,20 +773,20 @@ public class PrimaryKeyRestrictionSetTest // (clustering_0, clustering_1) = (1, 2) AND clustering_2 > 3 Restriction multiEq = newMultiEq(cfMetaData, 0, value1, value2); Restriction singleSlice = newSingleSlice(cfMetaData, 2, Bound.START, false, value3); - PrimaryKeyRestrictions restrictions = new PrimaryKeyRestrictionSet(cfMetaData.comparator); + PrimaryKeyRestrictions restrictions = new PrimaryKeyRestrictionSet(cfMetaData.comparator, false); restrictions = restrictions.mergeWith(multiEq).mergeWith(singleSlice); - List bounds = restrictions.boundsAsComposites(Bound.START, QueryOptions.DEFAULT); + SortedSet bounds = restrictions.boundsAsClustering(Bound.START, QueryOptions.DEFAULT); assertEquals(1, bounds.size()); - assertComposite(bounds.get(0), value1, value2, value3, EOC.END); + assertStartBound(get(bounds, 0), false, value1, value2, value3); - bounds = restrictions.boundsAsComposites(Bound.END, QueryOptions.DEFAULT); + bounds = restrictions.boundsAsClustering(Bound.END, QueryOptions.DEFAULT); assertEquals(1, bounds.size()); - assertComposite(bounds.get(0), value1, value2, EOC.END); + assertEndBound(get(bounds, 0), true, value1, value2); } @Test - public void testBoundsAsCompositesWithSeveralMultiColumnRestrictions() throws InvalidRequestException + public void testboundsAsClusteringWithSeveralMultiColumnRestrictions() throws InvalidRequestException { CFMetaData cfMetaData = newCFMetaData(4); @@ -803,142 +799,106 @@ public class PrimaryKeyRestrictionSetTest // (clustering_0, clustering_1) = (1, 2) AND (clustering_2, clustering_3) > (3, 4) Restriction multiEq = newMultiEq(cfMetaData, 0, value1, value2); Restriction multiSlice = newMultiSlice(cfMetaData, 2, Bound.START, false, value3, value4); - PrimaryKeyRestrictions restrictions = new PrimaryKeyRestrictionSet(cfMetaData.comparator); + PrimaryKeyRestrictions restrictions = new PrimaryKeyRestrictionSet(cfMetaData.comparator, false); restrictions = restrictions.mergeWith(multiEq).mergeWith(multiSlice); - List bounds = restrictions.boundsAsComposites(Bound.START, QueryOptions.DEFAULT); + SortedSet bounds = restrictions.boundsAsClustering(Bound.START, QueryOptions.DEFAULT); assertEquals(1, bounds.size()); - assertComposite(bounds.get(0), value1, value2, value3, value4, EOC.END); + assertStartBound(get(bounds, 0), false, value1, value2, value3, value4); - bounds = restrictions.boundsAsComposites(Bound.END, QueryOptions.DEFAULT); + bounds = restrictions.boundsAsClustering(Bound.END, QueryOptions.DEFAULT); assertEquals(1, bounds.size()); - assertComposite(bounds.get(0), value1, value2, EOC.END); + assertEndBound(get(bounds, 0), true, value1, value2); // (clustering_0, clustering_1) = (1, 2) AND (clustering_2, clustering_3) IN ((3, 4), (4, 5)) multiEq = newMultiEq(cfMetaData, 0, value1, value2); Restriction multiIN = newMultiIN(cfMetaData, 2, asList(value3, value4), asList(value4, value5)); - restrictions = new PrimaryKeyRestrictionSet(cfMetaData.comparator); + restrictions = new PrimaryKeyRestrictionSet(cfMetaData.comparator, false); restrictions = restrictions.mergeWith(multiEq).mergeWith(multiIN); - bounds = restrictions.boundsAsComposites(Bound.START, QueryOptions.DEFAULT); + bounds = restrictions.boundsAsClustering(Bound.START, QueryOptions.DEFAULT); assertEquals(2, bounds.size()); - assertComposite(bounds.get(0), value1, value2, value3, value4, EOC.START); - assertComposite(bounds.get(1), value1, value2, value4, value5, EOC.START); + assertStartBound(get(bounds, 0), true, value1, value2, value3, value4); + assertStartBound(get(bounds, 1), true, value1, value2, value4, value5); - bounds = restrictions.boundsAsComposites(Bound.END, QueryOptions.DEFAULT); + bounds = restrictions.boundsAsClustering(Bound.END, QueryOptions.DEFAULT); assertEquals(2, bounds.size()); - assertComposite(bounds.get(0), value1, value2, value3, value4, EOC.END); - assertComposite(bounds.get(1), value1, value2, value4, value5, EOC.END); + assertEndBound(get(bounds, 0), true, value1, value2, value3, value4); + assertEndBound(get(bounds, 1), true, value1, value2, value4, value5); // (clustering_0, clustering_1) = (1, 2) AND (clustering_2, clustering_3) = (3, 4) multiEq = newMultiEq(cfMetaData, 0, value1, value2); Restriction multiEq2 = newMultiEq(cfMetaData, 2, value3, value4); - restrictions = new PrimaryKeyRestrictionSet(cfMetaData.comparator); + restrictions = new PrimaryKeyRestrictionSet(cfMetaData.comparator, false); restrictions = restrictions.mergeWith(multiEq).mergeWith(multiEq2); - bounds = restrictions.boundsAsComposites(Bound.START, QueryOptions.DEFAULT); + bounds = restrictions.boundsAsClustering(Bound.START, QueryOptions.DEFAULT); assertEquals(1, bounds.size()); - assertComposite(bounds.get(0), value1, value2, value3, value4, EOC.START); + assertStartBound(get(bounds, 0), true, value1, value2, value3, value4); - bounds = restrictions.boundsAsComposites(Bound.END, QueryOptions.DEFAULT); + bounds = restrictions.boundsAsClustering(Bound.END, QueryOptions.DEFAULT); assertEquals(1, bounds.size()); - assertComposite(bounds.get(0), value1, value2, value3, value4, EOC.END); + assertEndBound(get(bounds, 0), true, value1, value2, value3, value4); } /** - * Asserts that the specified Composite is an empty one. + * Asserts that the specified Bound is an empty start. * - * @param composite the composite to check + * @param bound the bound to check */ - private static void assertEmptyComposite(Composite composite) + private static void assertEmptyStart(Slice.Bound bound) { - assertEquals(Composites.EMPTY, composite); + assertEquals(Slice.Bound.BOTTOM, bound); } /** - * Asserts that the specified Composite contains the specified element and the specified EOC. + * Asserts that the specified Bound is an empty end. * - * @param composite the composite to check - * @param element the expected element of the composite - * @param eoc the expected EOC of the composite + * @param bound the bound to check */ - private static void assertComposite(Composite composite, ByteBuffer element, EOC eoc) + private static void assertEmptyEnd(Slice.Bound bound) { - assertComposite(composite, eoc, element); + assertEquals(Slice.Bound.TOP, bound); } /** - * Asserts that the specified Composite contains the 2 specified element and the specified EOC. + * Asserts that the specified Slice.Bound is a start with the specified elements. * - * @param composite the composite to check - * @param eoc the expected EOC of the composite - * @param elements the expected element of the composite + * @param bound the bound to check + * @param isInclusive if the bound is expected to be inclusive + * @param elements the expected elements of the clustering */ - private static void assertComposite(Composite composite, ByteBuffer firstElement, ByteBuffer secondElement, EOC eoc) + private static void assertStartBound(Slice.Bound bound, boolean isInclusive, ByteBuffer... elements) { - assertComposite(composite, eoc, firstElement, secondElement); + assertBound(bound, true, isInclusive, elements); } /** - * Asserts that the specified Composite contains the 3 specified element and the specified EOC. + * Asserts that the specified Slice.Bound is a end with the specified elements. * - * @param composite the composite to check - * @param firstElement the first expected element of the composite - * @param secondElement the second expected element of the composite - * @param thirdElement the third expected element of the composite - * @param eoc the expected EOC of the composite - * @param elements the expected element of the composite + * @param bound the bound to check + * @param isInclusive if the bound is expected to be inclusive + * @param elements the expected elements of the clustering */ - private static void assertComposite(Composite composite, - ByteBuffer firstElement, - ByteBuffer secondElement, - ByteBuffer thirdElement, - EOC eoc) + private static void assertEndBound(Slice.Bound bound, boolean isInclusive, ByteBuffer... elements) { - assertComposite(composite, eoc, firstElement, secondElement, thirdElement); + assertBound(bound, false, isInclusive, elements); } - /** - * Asserts that the specified Composite contains the 4 specified element and the specified EOC. - * - * @param composite the composite to check - * @param firstElement the first expected element of the composite - * @param secondElement the second expected element of the composite - * @param thirdElement the third expected element of the composite - * @param fourthElement the fourth expected element of the composite - * @param eoc the expected EOC of the composite - * @param elements the expected element of the composite - */ - private static void assertComposite(Composite composite, - ByteBuffer firstElement, - ByteBuffer secondElement, - ByteBuffer thirdElement, - ByteBuffer fourthElement, - EOC eoc) + private static void assertBound(Slice.Bound bound, boolean isStart, boolean isInclusive, ByteBuffer... elements) { - assertComposite(composite, eoc, firstElement, secondElement, thirdElement, fourthElement); - } - - /** - * Asserts that the specified Composite contains the specified elements and EOC. - * - * @param composite the composite to check - * @param eoc the expected EOC of the composite - * @param elements the expected elements of the composite - */ - private static void assertComposite(Composite composite, EOC eoc, ByteBuffer... elements) - { - assertEquals("the composite size is not the expected one:", elements.length, composite.size()); + assertEquals("the bound size is not the expected one:", elements.length, bound.size()); + assertEquals("the bound should be a " + (isStart ? "start" : "end") + " but is a " + (bound.isStart() ? "start" : "end"), isStart, bound.isStart()); + assertEquals("the bound inclusiveness is not the expected one", isInclusive, bound.isInclusive()); for (int i = 0, m = elements.length; i < m; i++) { ByteBuffer element = elements[i]; - assertTrue(String.format("the element %s of the composite is not the expected one: expected %s but was %s", + assertTrue(String.format("the element %s of the bound is not the expected one: expected %s but was %s", i, ByteBufferUtil.toInt(element), - ByteBufferUtil.toInt(composite.get(i))), - element.equals(composite.get(i))); + ByteBufferUtil.toInt(bound.get(i))), + element.equals(bound.get(i))); } - assertEquals("the EOC of the composite is not the expected one:", eoc, composite.eoc()); } /** @@ -954,17 +914,13 @@ public class PrimaryKeyRestrictionSetTest for (int i = 0; i < numberOfClusteringColumns; i++) types.add(Int32Type.instance); - CompoundSparseCellNameType cType = new CompoundSparseCellNameType(types); - CFMetaData cfMetaData = new CFMetaData("keyspace", "test", ColumnFamilyType.Standard, cType); + CFMetaData.Builder builder = CFMetaData.Builder.create("keyspace", "test") + .addPartitionKey("partition_key", Int32Type.instance); for (int i = 0; i < numberOfClusteringColumns; i++) - { - ByteBuffer name = ByteBufferUtil.bytes("clustering_" + i); - ColumnDefinition columnDef = ColumnDefinition.clusteringKeyDef(cfMetaData, name, Int32Type.instance, i); - cfMetaData.addColumnDefinition(columnDef); - } - cfMetaData.rebuild(); - return cfMetaData; + builder.addClusteringColumn("clustering_" + i, Int32Type.instance); + + return builder.build(); } /** @@ -978,7 +934,7 @@ public class PrimaryKeyRestrictionSetTest private static Restriction newSingleEq(CFMetaData cfMetaData, int index, ByteBuffer value) { ColumnDefinition columnDef = getClusteringColumnDefinition(cfMetaData, index); - return new SingleColumnRestriction.EQ(columnDef, toTerm(value)); + return new SingleColumnRestriction.EQRestriction(columnDef, toTerm(value)); } /** @@ -996,7 +952,7 @@ public class PrimaryKeyRestrictionSetTest { columnDefinitions.add(getClusteringColumnDefinition(cfMetaData, firstIndex + i)); } - return new MultiColumnRestriction.EQ(columnDefinitions, toMultiItemTerminal(values)); + return new MultiColumnRestriction.EQRestriction(columnDefinitions, toMultiItemTerminal(values)); } /** @@ -1017,7 +973,7 @@ public class PrimaryKeyRestrictionSetTest columnDefinitions.add(getClusteringColumnDefinition(cfMetaData, firstIndex + i)); terms.add(toMultiItemTerminal(values[i].toArray(new ByteBuffer[0]))); } - return new MultiColumnRestriction.InWithValues(columnDefinitions, terms); + return new MultiColumnRestriction.InRestrictionWithValues(columnDefinitions, terms); } /** @@ -1031,7 +987,7 @@ public class PrimaryKeyRestrictionSetTest private static Restriction newSingleIN(CFMetaData cfMetaData, int index, ByteBuffer... values) { ColumnDefinition columnDef = getClusteringColumnDefinition(cfMetaData, index); - return new SingleColumnRestriction.InWithValues(columnDef, toTerms(values)); + return new SingleColumnRestriction.InRestrictionWithValues(columnDef, toTerms(values)); } /** @@ -1059,7 +1015,7 @@ public class PrimaryKeyRestrictionSetTest private static Restriction newSingleSlice(CFMetaData cfMetaData, int index, Bound bound, boolean inclusive, ByteBuffer value) { ColumnDefinition columnDef = getClusteringColumnDefinition(cfMetaData, index); - return new SingleColumnRestriction.Slice(columnDef, bound, inclusive, toTerm(value)); + return new SingleColumnRestriction.SliceRestriction(columnDef, bound, inclusive, toTerm(value)); } /** @@ -1079,7 +1035,7 @@ public class PrimaryKeyRestrictionSetTest { columnDefinitions.add(getClusteringColumnDefinition(cfMetaData, i + firstIndex)); } - return new MultiColumnRestriction.Slice(columnDefinitions, bound, inclusive, toMultiItemTerminal(values)); + return new MultiColumnRestriction.SliceRestriction(columnDefinitions, bound, inclusive, toMultiItemTerminal(values)); } /** @@ -1117,4 +1073,9 @@ public class PrimaryKeyRestrictionSetTest terms.add(toTerm(value)); return terms; } + + private static T get(SortedSet set, int i) + { + return Iterables.get(set, i); + } } diff --git a/test/unit/org/apache/cassandra/cql3/validation/entities/CollectionsTest.java b/test/unit/org/apache/cassandra/cql3/validation/entities/CollectionsTest.java index 2e72c398fd..72f5ad542d 100644 --- a/test/unit/org/apache/cassandra/cql3/validation/entities/CollectionsTest.java +++ b/test/unit/org/apache/cassandra/cql3/validation/entities/CollectionsTest.java @@ -237,7 +237,7 @@ public class CollectionsTest extends CQLTester assertInvalidMessage("Attempted to set an element on a list which is null", "UPDATE %s SET l[0] = ? WHERE k=0", list("v10")); - execute("UPDATE %s SET l = l - ? WHERE k=0 ", list("v11")); + execute("UPDATE %s SET l = l - ? WHERE k=0", list("v11")); assertRows(execute("SELECT l FROM %s WHERE k = 0"), row((Object) null)); } diff --git a/test/unit/org/apache/cassandra/cql3/validation/entities/CountersTest.java b/test/unit/org/apache/cassandra/cql3/validation/entities/CountersTest.java index e5ff251492..e54d105e42 100644 --- a/test/unit/org/apache/cassandra/cql3/validation/entities/CountersTest.java +++ b/test/unit/org/apache/cassandra/cql3/validation/entities/CountersTest.java @@ -59,8 +59,8 @@ public class CountersTest extends CQLTester @Test public void testRegularCounters() throws Throwable { - assertInvalidThrowMessage("Cannot add a non counter column", - ConfigurationException.class, + assertInvalidThrowMessage("Cannot mix counter and non counter columns in the same table", + InvalidRequestException.class, String.format("CREATE TABLE %s.%s (id bigint PRIMARY KEY, count counter, things set)", KEYSPACE, createTableName())); } diff --git a/test/unit/org/apache/cassandra/cql3/validation/entities/FrozenCollectionsTest.java b/test/unit/org/apache/cassandra/cql3/validation/entities/FrozenCollectionsTest.java index 857139d024..c76d618a01 100644 --- a/test/unit/org/apache/cassandra/cql3/validation/entities/FrozenCollectionsTest.java +++ b/test/unit/org/apache/cassandra/cql3/validation/entities/FrozenCollectionsTest.java @@ -631,12 +631,9 @@ public class FrozenCollectionsTest extends CQLTester assertInvalidMessage("Cannot restrict clustering columns by a CONTAINS relation without a secondary index", "SELECT * FROM %s WHERE b CONTAINS ? ALLOW FILTERING", 1); - assertInvalidMessage("No secondary indexes on the restricted columns support the provided operator", + assertInvalidMessage("No supported secondary index found for the non primary key columns restrictions", "SELECT * FROM %s WHERE d CONTAINS KEY ?", 1); - assertInvalidMessage("No secondary indexes on the restricted columns support the provided operator", - "SELECT * FROM %s WHERE d CONTAINS KEY ? ALLOW FILTERING", 1); - assertInvalidMessage("Cannot restrict clustering columns by a CONTAINS relation without a secondary index", "SELECT * FROM %s WHERE b CONTAINS ? AND d CONTAINS KEY ? ALLOW FILTERING", 1, 1); @@ -747,6 +744,11 @@ public class FrozenCollectionsTest extends CQLTester row(0, list(1, 2, 3), set(1, 2, 3), map(1, "a")) ); + assertRows(execute("SELECT * FROM %s WHERE d CONTAINS KEY ? ALLOW FILTERING", 1), + row(0, list(1, 2, 3), set(1, 2, 3), map(1, "a")), + row(0, list(4, 5, 6), set(1, 2, 3), map(1, "a")) + ); + execute("DELETE d FROM %s WHERE a=? AND b=?", 0, list(1, 2, 3)); assertRows(execute("SELECT * FROM %s WHERE d=?", map(1, "a")), row(0, list(4, 5, 6), set(1, 2, 3), map(1, "a")) diff --git a/test/unit/org/apache/cassandra/cql3/validation/entities/SecondaryIndexTest.java b/test/unit/org/apache/cassandra/cql3/validation/entities/SecondaryIndexTest.java index 0b812c60c7..2c30b70227 100644 --- a/test/unit/org/apache/cassandra/cql3/validation/entities/SecondaryIndexTest.java +++ b/test/unit/org/apache/cassandra/cql3/validation/entities/SecondaryIndexTest.java @@ -97,7 +97,7 @@ public class SecondaryIndexTest extends CQLTester execute("DROP INDEX " + indexName); } - assertInvalidMessage("No secondary indexes on the restricted columns support the provided operators", + assertInvalidMessage("No supported secondary index found for the non primary key columns restrictions", "SELECT * FROM %s where b = ?", 1); dropIndex("DROP INDEX IF EXISTS " + indexName); assertInvalidMessage("Index '" + removeQuotes(indexName.toLowerCase(Locale.US)) + "' could not be found", "DROP INDEX " + indexName); @@ -483,23 +483,6 @@ public class SecondaryIndexTest extends CQLTester failInsert("INSERT INTO %s (a, b, c) VALUES (0, 0, ?)", ByteBuffer.allocate(TOO_BIG)); } - @Test - public void testIndexOnClusteringColumnInsertPartitionKeyAndClusteringsOver64k() throws Throwable - { - createTable("CREATE TABLE %s(a blob, b blob, c blob, d int, PRIMARY KEY (a, b, c))"); - createIndex("CREATE INDEX ON %s(b)"); - - // CompositeIndexOnClusteringKey creates index entries composed of the - // PK plus all of the non-indexed clustering columns from the primary row - // so we should reject where len(a) + len(c) > 65560 as this will form the - // total clustering in the index table - ByteBuffer a = ByteBuffer.allocate(100); - ByteBuffer b = ByteBuffer.allocate(10); - ByteBuffer c = ByteBuffer.allocate(FBUtilities.MAX_UNSIGNED_SHORT - 99); - - failInsert("INSERT INTO %s (a, b, c, d) VALUES (?, ?, ?, 0)", a, b, c); - } - @Test public void testCompactTableWithValueOver64k() throws Throwable { @@ -508,40 +491,6 @@ public class SecondaryIndexTest extends CQLTester failInsert("INSERT INTO %s (a, b) VALUES (0, ?)", ByteBuffer.allocate(TOO_BIG)); } - @Test - public void testIndexOnCollectionValueInsertPartitionKeyAndCollectionKeyOver64k() throws Throwable - { - createTable("CREATE TABLE %s(a blob , b map, PRIMARY KEY (a))"); - createIndex("CREATE INDEX ON %s(b)"); - - // A collection key > 64k by itself will be rejected from - // the primary table. - // To test index validation we need to ensure that - // len(b) < 64k, but len(a) + len(b) > 64k as that will - // form the clustering in the index table - ByteBuffer a = ByteBuffer.allocate(100); - ByteBuffer b = ByteBuffer.allocate(FBUtilities.MAX_UNSIGNED_SHORT - 100); - - failInsert("UPDATE %s SET b[?] = 0 WHERE a = ?", b, a); - } - - @Test - public void testIndexOnCollectionKeyInsertPartitionKeyAndClusteringOver64k() throws Throwable - { - createTable("CREATE TABLE %s(a blob, b blob, c map, PRIMARY KEY (a, b))"); - createIndex("CREATE INDEX ON %s(KEYS(c))"); - - // Basically the same as the case with non-collection clustering - // CompositeIndexOnCollectionKeyy creates index entries composed of the - // PK plus all of the clustering columns from the primary row, except the - // collection element - which becomes the partition key in the index table - ByteBuffer a = ByteBuffer.allocate(100); - ByteBuffer b = ByteBuffer.allocate(FBUtilities.MAX_UNSIGNED_SHORT - 100); - ByteBuffer c = ByteBuffer.allocate(10); - - failInsert("UPDATE %s SET c[?] = 0 WHERE a = ? and b = ?", c, a, b); - } - @Test public void testIndexOnPartitionKeyInsertValueOver64k() throws Throwable { diff --git a/test/unit/org/apache/cassandra/cql3/validation/entities/UFAuthTest.java b/test/unit/org/apache/cassandra/cql3/validation/entities/UFAuthTest.java index 498f0dd8e5..ee20557081 100644 --- a/test/unit/org/apache/cassandra/cql3/validation/entities/UFAuthTest.java +++ b/test/unit/org/apache/cassandra/cql3/validation/entities/UFAuthTest.java @@ -265,7 +265,7 @@ public class UFAuthTest extends CQLTester { setupTable("CREATE TABLE %s (k int, s int STATIC, v1 int, v2 int, PRIMARY KEY(k, v1))"); String functionName = createSimpleFunction(); - String cql = String.format("SELECT k FROM %s WHERE k = 0 AND s = %s", + String cql = String.format("SELECT k FROM %s WHERE k = 0 AND s = %s ALLOW FILTERING", KEYSPACE + "." + currentTable(), functionCall(functionName)); assertPermissionsOnFunction(cql, functionName); diff --git a/test/unit/org/apache/cassandra/cql3/validation/entities/UFIdentificationTest.java b/test/unit/org/apache/cassandra/cql3/validation/entities/UFIdentificationTest.java index 28b8afcf2a..8ab8a234cd 100644 --- a/test/unit/org/apache/cassandra/cql3/validation/entities/UFIdentificationTest.java +++ b/test/unit/org/apache/cassandra/cql3/validation/entities/UFIdentificationTest.java @@ -200,13 +200,13 @@ public class UFIdentificationTest extends CQLTester public void testSelectStatementSimpleRestrictions() throws Throwable { assertFunctions(cql("SELECT i_val FROM %s WHERE key=%s", functionCall(iFunc, "1")), iFunc); - assertFunctions(cql("SELECT i_val FROM %s WHERE key=0 AND t_sc=%s", functionCall(tFunc, "'foo'")), tFunc); - assertFunctions(cql("SELECT i_val FROM %s WHERE key=0 AND i_cc=%s AND t_cc='foo'", functionCall(iFunc, "1")), iFunc); - assertFunctions(cql("SELECT i_val FROM %s WHERE key=0 AND i_cc=0 AND t_cc=%s", functionCall(tFunc, "'foo'")), tFunc); + assertFunctions(cql("SELECT i_val FROM %s WHERE key=0 AND t_sc=%s ALLOW FILTERING", functionCall(tFunc, "'foo'")), tFunc); + assertFunctions(cql("SELECT i_val FROM %s WHERE key=0 AND i_cc=%s AND t_cc='foo' ALLOW FILTERING", functionCall(iFunc, "1")), iFunc); + assertFunctions(cql("SELECT i_val FROM %s WHERE key=0 AND i_cc=0 AND t_cc=%s ALLOW FILTERING", functionCall(tFunc, "'foo'")), tFunc); String iFunc2 = createEchoFunction("int"); String tFunc2 = createEchoFunction("text"); - assertFunctions(cql("SELECT i_val FROM %s WHERE key=%s AND t_sc=%s AND i_cc=%s AND t_cc=%s", + assertFunctions(cql("SELECT i_val FROM %s WHERE key=%s AND t_sc=%s AND i_cc=%s AND t_cc=%s ALLOW FILTERING", functionCall(iFunc, "1"), functionCall(tFunc, "'foo'"), functionCall(iFunc2, "1"), diff --git a/test/unit/org/apache/cassandra/cql3/validation/entities/UFTest.java b/test/unit/org/apache/cassandra/cql3/validation/entities/UFTest.java index 2e7c2f1ac1..7bd208fb46 100644 --- a/test/unit/org/apache/cassandra/cql3/validation/entities/UFTest.java +++ b/test/unit/org/apache/cassandra/cql3/validation/entities/UFTest.java @@ -48,6 +48,7 @@ import org.apache.cassandra.dht.ByteOrderedPartitioner; import org.apache.cassandra.exceptions.FunctionExecutionException; import org.apache.cassandra.exceptions.InvalidRequestException; import org.apache.cassandra.service.ClientState; +import org.apache.cassandra.service.StorageService; import org.apache.cassandra.transport.Event; import org.apache.cassandra.transport.Server; import org.apache.cassandra.transport.messages.ResultMessage; @@ -59,7 +60,7 @@ public class UFTest extends CQLTester @BeforeClass public static void setUp() { - DatabaseDescriptor.setPartitioner(ByteOrderedPartitioner.instance); + StorageService.instance.setPartitionerUnsafe(ByteOrderedPartitioner.instance); } @Test diff --git a/test/unit/org/apache/cassandra/cql3/validation/miscellaneous/CrcCheckChanceTest.java b/test/unit/org/apache/cassandra/cql3/validation/miscellaneous/CrcCheckChanceTest.java index 98d7d7021a..097a77dc79 100644 --- a/test/unit/org/apache/cassandra/cql3/validation/miscellaneous/CrcCheckChanceTest.java +++ b/test/unit/org/apache/cassandra/cql3/validation/miscellaneous/CrcCheckChanceTest.java @@ -69,8 +69,6 @@ public class CrcCheckChanceTest extends CQLTester row("p1", "k1", "sv1", "v1") ); - - //Write a few SSTables then Compact execute("INSERT INTO %s(p, c, v, s) values (?, ?, ?, ?)", "p1", "k1", "v1", "sv1"); @@ -143,7 +141,7 @@ public class CrcCheckChanceTest extends CQLTester } DatabaseDescriptor.setCompactionThroughputMbPerSec(1); - List> futures = CompactionManager.instance.submitMaximal(cfs, CompactionManager.getDefaultGcBefore(cfs), false); + List> futures = CompactionManager.instance.submitMaximal(cfs, CompactionManager.getDefaultGcBefore(cfs, FBUtilities.nowInSeconds()), false); execute("DROP TABLE %s"); try diff --git a/test/unit/org/apache/cassandra/cql3/validation/miscellaneous/SSTableMetadataTrackingTest.java b/test/unit/org/apache/cassandra/cql3/validation/miscellaneous/SSTableMetadataTrackingTest.java index 2a2ca7bc27..89f80f5119 100644 --- a/test/unit/org/apache/cassandra/cql3/validation/miscellaneous/SSTableMetadataTrackingTest.java +++ b/test/unit/org/apache/cassandra/cql3/validation/miscellaneous/SSTableMetadataTrackingTest.java @@ -140,6 +140,7 @@ public class SSTableMetadataTrackingTest extends CQLTester assertEquals(metadata.minTimestamp, metadata2.minTimestamp); assertEquals(metadata.maxTimestamp, metadata2.maxTimestamp); } + @Test public void testTrackMetadata_rowMarkerDelete() throws Throwable { diff --git a/test/unit/org/apache/cassandra/cql3/validation/operations/AlterTest.java b/test/unit/org/apache/cassandra/cql3/validation/operations/AlterTest.java index 95380f4a2f..05f6c35240 100644 --- a/test/unit/org/apache/cassandra/cql3/validation/operations/AlterTest.java +++ b/test/unit/org/apache/cassandra/cql3/validation/operations/AlterTest.java @@ -48,6 +48,7 @@ public class AlterTest extends CQLTester assertRows(execute("SELECT * FROM %s;"), row("test", "first test")); } + @Test public void testAddMap() throws Throwable { @@ -80,6 +81,7 @@ public class AlterTest extends CQLTester execute("UPDATE %s set myCollection = ['second element'] WHERE id = 'test';"); assertRows(execute("SELECT * FROM %s;"), row("test", "first test", list("second element"))); } + @Test public void testDropListAndAddMapWithSameName() throws Throwable { diff --git a/test/unit/org/apache/cassandra/cql3/validation/operations/CreateTest.java b/test/unit/org/apache/cassandra/cql3/validation/operations/CreateTest.java index 398b851711..18f2db4db7 100644 --- a/test/unit/org/apache/cassandra/cql3/validation/operations/CreateTest.java +++ b/test/unit/org/apache/cassandra/cql3/validation/operations/CreateTest.java @@ -30,8 +30,8 @@ import org.apache.cassandra.config.CFMetaData; import org.apache.cassandra.config.Schema; import org.apache.cassandra.config.TriggerDefinition; import org.apache.cassandra.cql3.CQLTester; -import org.apache.cassandra.db.ColumnFamily; import org.apache.cassandra.db.Mutation; +import org.apache.cassandra.db.partitions.Partition; import org.apache.cassandra.exceptions.ConfigurationException; import org.apache.cassandra.exceptions.SyntaxException; import org.apache.cassandra.triggers.ITrigger; @@ -466,10 +466,10 @@ public class CreateTest extends CQLTester { createTable("CREATE TABLE %s (a int, b int , c int, PRIMARY KEY (a, b)) WITH COMPACT STORAGE;"); - assertInvalidMessage("Secondary indexes are not supported on PRIMARY KEY columns in COMPACT STORAGE tables", + assertInvalidMessage("Secondary indexes are not supported on COMPACT STORAGE tables that have clustering columns", "CREATE INDEX ON %s (a);"); - assertInvalidMessage("Secondary indexes are not supported on PRIMARY KEY columns in COMPACT STORAGE tables", + assertInvalidMessage("Secondary indexes are not supported on COMPACT STORAGE tables that have clustering columns", "CREATE INDEX ON %s (b);"); assertInvalidMessage("Secondary indexes are not supported on COMPACT STORAGE tables that have clustering columns", @@ -522,7 +522,7 @@ public class CreateTest extends CQLTester public static class TestTrigger implements ITrigger { public TestTrigger() { } - public Collection augment(ByteBuffer key, ColumnFamily update) + public Collection augment(Partition update) { return Collections.emptyList(); } diff --git a/test/unit/org/apache/cassandra/cql3/validation/operations/SelectLimitTest.java b/test/unit/org/apache/cassandra/cql3/validation/operations/SelectLimitTest.java index 275ff04b10..f1e2f557dc 100644 --- a/test/unit/org/apache/cassandra/cql3/validation/operations/SelectLimitTest.java +++ b/test/unit/org/apache/cassandra/cql3/validation/operations/SelectLimitTest.java @@ -105,8 +105,12 @@ public class SelectLimitTest extends CQLTester row(1, 1), row(1, 2), row(1, 3)); - - // strict bound (v > 1) over a range of partitions is not supported for compact storage if limit is provided - assertInvalidThrow(InvalidRequestException.class, "SELECT * FROM %s WHERE v > 1 AND v <= 3 LIMIT 6 ALLOW FILTERING"); + assertRows(execute("SELECT * FROM %s WHERE v > 1 AND v <= 3 LIMIT 6 ALLOW FILTERING"), + row(0, 2), + row(0, 3), + row(1, 2), + row(1, 3), + row(2, 2), + row(2, 3)); } } diff --git a/test/unit/org/apache/cassandra/cql3/validation/operations/SelectSingleColumnRelationTest.java b/test/unit/org/apache/cassandra/cql3/validation/operations/SelectSingleColumnRelationTest.java index 57b8a86ba2..ff73a956a6 100644 --- a/test/unit/org/apache/cassandra/cql3/validation/operations/SelectSingleColumnRelationTest.java +++ b/test/unit/org/apache/cassandra/cql3/validation/operations/SelectSingleColumnRelationTest.java @@ -142,7 +142,7 @@ public class SelectSingleColumnRelationTest extends CQLTester assertRows(execute("select * from %s where a = ? and c < ? and b in (?, ?)", "first", 7, 3, 2), row("first", 2, 6, 2)); -//--- + assertRows(execute("select * from %s where a = ? and c >= ? and c <= ? and b in (?, ?)", "first", 6, 7, 3, 2), row("first", 2, 6, 2), row("first", 3, 7, 3)); diff --git a/test/unit/org/apache/cassandra/cql3/validation/operations/SelectTest.java b/test/unit/org/apache/cassandra/cql3/validation/operations/SelectTest.java index 506bdafc81..8622c98501 100644 --- a/test/unit/org/apache/cassandra/cql3/validation/operations/SelectTest.java +++ b/test/unit/org/apache/cassandra/cql3/validation/operations/SelectTest.java @@ -605,7 +605,7 @@ public class SelectTest extends CQLTester execute("INSERT INTO %s (account, id , categories) VALUES (?, ?, ?)", "test", 5, map("lmn", "foo")); execute("INSERT INTO %s (account, id , categories) VALUES (?, ?, ?)", "test", 6, map("lmn", "foo2")); - assertInvalidMessage("No secondary indexes on the restricted columns support the provided operators: 'categories CONTAINS '", + assertInvalidMessage("No supported secondary index found for the non primary key columns restrictions", "SELECT * FROM %s WHERE account = ? AND categories CONTAINS ?", "test", "foo"); assertRows(execute("SELECT * FROM %s WHERE account = ? AND categories CONTAINS KEY ?", "test", "lmn"), @@ -628,7 +628,7 @@ public class SelectTest extends CQLTester execute("INSERT INTO %s (account, id , categories) VALUES (?, ?, ?)", "test", 5, map("lmn", "foo")); execute("INSERT INTO %s (account, id , categories) VALUES (?, ?, ?)", "test", 6, map("lmn2", "foo")); - assertInvalidMessage("No secondary indexes on the restricted columns support the provided operators: 'categories CONTAINS KEY '", + assertInvalidMessage("No supported secondary index found for the non primary key columns restrictions", "SELECT * FROM %s WHERE account = ? AND categories CONTAINS KEY ?", "test", "lmn"); assertRows(execute("SELECT * FROM %s WHERE account = ? AND categories CONTAINS ?", "test", "foo"), diff --git a/test/unit/org/apache/cassandra/db/ArrayBackedSortedColumnsTest.java b/test/unit/org/apache/cassandra/db/ArrayBackedSortedColumnsTest.java deleted file mode 100644 index 0fdabe9d73..0000000000 --- a/test/unit/org/apache/cassandra/db/ArrayBackedSortedColumnsTest.java +++ /dev/null @@ -1,426 +0,0 @@ -/* - * - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - * - */ -package org.apache.cassandra.db; - -import java.util.*; - -import org.junit.BeforeClass; -import org.junit.Test; - -import static org.junit.Assert.*; - -import com.google.common.collect.Sets; - -import org.apache.cassandra.SchemaLoader; -import org.apache.cassandra.config.CFMetaData; -import org.apache.cassandra.config.KSMetaData; -import org.apache.cassandra.config.Schema; -import org.apache.cassandra.db.composites.*; -import org.apache.cassandra.db.filter.ColumnSlice; -import org.apache.cassandra.db.marshal.Int32Type; -import org.apache.cassandra.exceptions.ConfigurationException; -import org.apache.cassandra.locator.SimpleStrategy; -import org.apache.cassandra.utils.ByteBufferUtil; -import org.apache.cassandra.utils.SearchIterator; -import org.apache.cassandra.utils.BatchRemoveIterator; - -public class ArrayBackedSortedColumnsTest -{ - private static final String KEYSPACE1 = "ArrayBackedSortedColumnsTest"; - private static final String CF_STANDARD1 = "Standard1"; - - @BeforeClass - public static void defineSchema() throws ConfigurationException - { - SchemaLoader.prepareServer(); - SchemaLoader.createKeyspace(KEYSPACE1, - SimpleStrategy.class, - KSMetaData.optsWithRF(1), - SchemaLoader.standardCFMD(KEYSPACE1, CF_STANDARD1)); - } - - @Test - public void testAdd() - { - testAddInternal(false); - testAddInternal(true); - } - - private CFMetaData metadata() - { - return Schema.instance.getCFMetaData(KEYSPACE1, CF_STANDARD1); - } - - private void testAddInternal(boolean reversed) - { - CellNameType type = new SimpleDenseCellNameType(Int32Type.instance); - ColumnFamily map = ArrayBackedSortedColumns.factory.create(metadata(), reversed); - int[] values = new int[]{ 1, 2, 2, 3 }; - - for (int i = 0; i < values.length; ++i) - map.addColumn(new BufferCell(type.makeCellName(values[reversed ? values.length - 1 - i : i]))); - - Iterator iter = map.iterator(); - assertEquals("1st column", 1, iter.next().name().toByteBuffer().getInt(0)); - assertEquals("2nd column", 2, iter.next().name().toByteBuffer().getInt(0)); - assertEquals("3rd column", 3, iter.next().name().toByteBuffer().getInt(0)); - } - - @Test - public void testOutOfOrder() - { - testAddOutOfOrder(false); - testAddOutOfOrder(false); - } - - private void testAddOutOfOrder(boolean reversed) - { - CellNameType type = new SimpleDenseCellNameType(Int32Type.instance); - ColumnFamily cells = ArrayBackedSortedColumns.factory.create(metadata(), reversed); - - int[] values = new int[]{ 1, 2, 1, 3, 4, 4, 5, 5, 1, 2, 6, 6, 6, 1, 2, 3 }; - for (int i = 0; i < values.length; ++i) - cells.addColumn(new BufferCell(type.makeCellName(values[reversed ? values.length - 1 - i : i]))); - - assertEquals(6, cells.getColumnCount()); - - Iterator iter = cells.iterator(); - assertEquals(1, iter.next().name().toByteBuffer().getInt(0)); - assertEquals(2, iter.next().name().toByteBuffer().getInt(0)); - assertEquals(3, iter.next().name().toByteBuffer().getInt(0)); - assertEquals(4, iter.next().name().toByteBuffer().getInt(0)); - assertEquals(5, iter.next().name().toByteBuffer().getInt(0)); - assertEquals(6, iter.next().name().toByteBuffer().getInt(0)); - - // Add more values - values = new int[]{ 11, 15, 12, 12, 12, 16, 10, 8, 8, 7, 4, 4, 5 }; - for (int i = 0; i < values.length; ++i) - cells.addColumn(new BufferCell(type.makeCellName(values[reversed ? values.length - 1 - i : i]))); - - assertEquals(13, cells.getColumnCount()); - - iter = cells.reverseIterator(); - assertEquals(16, iter.next().name().toByteBuffer().getInt(0)); - assertEquals(15, iter.next().name().toByteBuffer().getInt(0)); - assertEquals(12, iter.next().name().toByteBuffer().getInt(0)); - assertEquals(11, iter.next().name().toByteBuffer().getInt(0)); - assertEquals(10, iter.next().name().toByteBuffer().getInt(0)); - assertEquals(8, iter.next().name().toByteBuffer().getInt(0)); - assertEquals(7, iter.next().name().toByteBuffer().getInt(0)); - assertEquals(6, iter.next().name().toByteBuffer().getInt(0)); - assertEquals(5, iter.next().name().toByteBuffer().getInt(0)); - assertEquals(4, iter.next().name().toByteBuffer().getInt(0)); - assertEquals(3, iter.next().name().toByteBuffer().getInt(0)); - assertEquals(2, iter.next().name().toByteBuffer().getInt(0)); - assertEquals(1, iter.next().name().toByteBuffer().getInt(0)); - } - - @Test - public void testGetColumn() - { - testGetColumnInternal(true); - testGetColumnInternal(false); - } - - private void testGetColumnInternal(boolean reversed) - { - CellNameType type = new SimpleDenseCellNameType(Int32Type.instance); - ColumnFamily cells = ArrayBackedSortedColumns.factory.create(metadata(), reversed); - - int[] values = new int[]{ -1, 20, 44, 55, 27, 27, 17, 1, 9, 89, 33, 44, 0, 9 }; - for (int i = 0; i < values.length; ++i) - cells.addColumn(new BufferCell(type.makeCellName(values[reversed ? values.length - 1 - i : i]))); - - for (int i : values) - assertEquals(i, cells.getColumn(type.makeCellName(i)).name().toByteBuffer().getInt(0)); - } - - @Test - public void testAddAll() - { - testAddAllInternal(false); - testAddAllInternal(true); - } - - private void testAddAllInternal(boolean reversed) - { - CellNameType type = new SimpleDenseCellNameType(Int32Type.instance); - ColumnFamily map = ArrayBackedSortedColumns.factory.create(metadata(), reversed); - ColumnFamily map2 = ArrayBackedSortedColumns.factory.create(metadata(), reversed); - - int[] values1 = new int[]{ 1, 3, 5, 6 }; - int[] values2 = new int[]{ 2, 4, 5, 6 }; - - for (int i = 0; i < values1.length; ++i) - map.addColumn(new BufferCell(type.makeCellName(values1[reversed ? values1.length - 1 - i : i]))); - - for (int i = 0; i < values2.length; ++i) - map2.addColumn(new BufferCell(type.makeCellName(values2[reversed ? values2.length - 1 - i : i]))); - - map2.addAll(map); - - Iterator iter = map2.iterator(); - assertEquals("1st column", 1, iter.next().name().toByteBuffer().getInt(0)); - assertEquals("2nd column", 2, iter.next().name().toByteBuffer().getInt(0)); - assertEquals("3rd column", 3, iter.next().name().toByteBuffer().getInt(0)); - assertEquals("4st column", 4, iter.next().name().toByteBuffer().getInt(0)); - assertEquals("5st column", 5, iter.next().name().toByteBuffer().getInt(0)); - assertEquals("6st column", 6, iter.next().name().toByteBuffer().getInt(0)); - } - - @Test - public void testGetCollection() - { - testGetCollectionInternal(false); - testGetCollectionInternal(true); - } - - private void testGetCollectionInternal(boolean reversed) - { - CellNameType type = new SimpleDenseCellNameType(Int32Type.instance); - ColumnFamily map = ArrayBackedSortedColumns.factory.create(metadata(), reversed); - int[] values = new int[]{ 1, 2, 3, 5, 9 }; - - List sorted = new ArrayList<>(); - for (int v : values) - sorted.add(new BufferCell(type.makeCellName(v))); - List reverseSorted = new ArrayList<>(sorted); - Collections.reverse(reverseSorted); - - for (int i = 0; i < values.length; ++i) - map.addColumn(new BufferCell(type.makeCellName(values[reversed ? values.length - 1 - i : i]))); - - assertSame(sorted, map.getSortedColumns()); - assertSame(reverseSorted, map.getReverseSortedColumns()); - } - - @Test - public void testIterator() - { - testIteratorInternal(false); - //testIteratorInternal(true); - } - - private void testIteratorInternal(boolean reversed) - { - CellNameType type = new SimpleDenseCellNameType(Int32Type.instance); - ColumnFamily map = ArrayBackedSortedColumns.factory.create(metadata(), reversed); - - int[] values = new int[]{ 1, 2, 3, 5, 9 }; - - for (int i = 0; i < values.length; ++i) - map.addColumn(new BufferCell(type.makeCellName(values[reversed ? values.length - 1 - i : i]))); - - assertSame(new int[]{ 3, 2, 1 }, map.reverseIterator(new ColumnSlice[]{ new ColumnSlice(type.make(3), Composites.EMPTY) })); - assertSame(new int[]{ 3, 2, 1 }, map.reverseIterator(new ColumnSlice[]{ new ColumnSlice(type.make(4), Composites.EMPTY) })); - - assertSame(map.iterator(), map.iterator(ColumnSlice.ALL_COLUMNS_ARRAY)); - } - - @Test - public void testSearchIterator() - { - CellNameType type = new SimpleDenseCellNameType(Int32Type.instance); - ColumnFamily map = ArrayBackedSortedColumns.factory.create(metadata(), false); - - int[] values = new int[]{ 1, 2, 3, 5, 9, 15, 21, 22 }; - - for (int i = 0; i < values.length; ++i) - map.addColumn(new BufferCell(type.makeCellName(values[i]))); - - SearchIterator iter = map.searchIterator(); - for (int i = 0 ; i < values.length ; i++) - assertSame(values[i], iter.next(type.makeCellName(values[i]))); - - iter = map.searchIterator(); - for (int i = 0 ; i < values.length ; i+=2) - assertSame(values[i], iter.next(type.makeCellName(values[i]))); - - iter = map.searchIterator(); - for (int i = 0 ; i < values.length ; i+=4) - assertSame(values[i], iter.next(type.makeCellName(values[i]))); - - iter = map.searchIterator(); - for (int i = 0 ; i < values.length ; i+=1) - { - if (i % 2 == 0) - { - Cell cell = iter.next(type.makeCellName(values[i] - 1)); - if (i > 0 && values[i - 1] == values[i] - 1) - assertSame(values[i - 1], cell); - else - assertNull(cell); - } - } - } - - private void assertSame(Iterable c1, Iterable c2) - { - assertSame(c1.iterator(), c2.iterator()); - } - - private void assertSame(Iterator iter1, Iterator iter2) - { - while (iter1.hasNext() && iter2.hasNext()) - assertEquals(iter1.next(), iter2.next()); - if (iter1.hasNext() || iter2.hasNext()) - fail("The collection don't have the same size"); - } - - private void assertSame(int name, Cell cell) - { - int value = ByteBufferUtil.toInt(cell.name().toByteBuffer()); - assert name == value : "Expected " + name + " but got " + value; - } - private void assertSame(int[] names, Iterator iter) - { - for (int name : names) - { - assert iter.hasNext() : "Expected " + name + " but no more result"; - int value = ByteBufferUtil.toInt(iter.next().name().toByteBuffer()); - assert name == value : "Expected " + name + " but got " + value; - } - } - - @Test - public void testRemove() - { - testRemoveInternal(false); - testRemoveInternal(true); - } - - private void testRemoveInternal(boolean reversed) - { - CellNameType type = new SimpleDenseCellNameType(Int32Type.instance); - ColumnFamily map = ArrayBackedSortedColumns.factory.create(metadata(), reversed); - - int[] values = new int[]{ 1, 2, 2, 3 }; - - for (int i = 0; i < values.length; ++i) - map.addColumn(new BufferCell(type.makeCellName(values[reversed ? values.length - 1 - i : i]))); - - Iterator iter = map.getReverseSortedColumns().iterator(); - assertTrue(iter.hasNext()); - iter.next(); - iter.remove(); - assertTrue(iter.hasNext()); - iter.next(); - iter.remove(); - assertTrue(iter.hasNext()); - iter.next(); - iter.remove(); - assertTrue(!iter.hasNext()); - } - - @Test(expected = IllegalStateException.class) - public void testBatchRemoveTwice() - { - CellNameType type = new SimpleDenseCellNameType(Int32Type.instance); - ColumnFamily map = ArrayBackedSortedColumns.factory.create(metadata(), false); - map.addColumn(new BufferCell(type.makeCellName(1))); - map.addColumn(new BufferCell(type.makeCellName(2))); - - BatchRemoveIterator batchIter = map.batchRemoveIterator(); - batchIter.next(); - batchIter.remove(); - batchIter.remove(); - } - - @Test(expected = IllegalStateException.class) - public void testBatchCommitTwice() - { - CellNameType type = new SimpleDenseCellNameType(Int32Type.instance); - ColumnFamily map = ArrayBackedSortedColumns.factory.create(metadata(), false); - map.addColumn(new BufferCell(type.makeCellName(1))); - map.addColumn(new BufferCell(type.makeCellName(2))); - - BatchRemoveIterator batchIter = map.batchRemoveIterator(); - batchIter.next(); - batchIter.remove(); - batchIter.commit(); - batchIter.commit(); - } - - @Test - public void testBatchRemove() - { - testBatchRemoveInternal(false); - testBatchRemoveInternal(true); - } - - public void testBatchRemoveInternal(boolean reversed) - { - CellNameType type = new SimpleDenseCellNameType(Int32Type.instance); - ColumnFamily map = ArrayBackedSortedColumns.factory.create(metadata(), reversed); - int[] values = new int[]{ 1, 2, 3, 5 }; - - for (int i = 0; i < values.length; ++i) - map.addColumn(new BufferCell(type.makeCellName(values[reversed ? values.length - 1 - i : i]))); - - BatchRemoveIterator batchIter = map.batchRemoveIterator(); - batchIter.next(); - batchIter.remove(); - batchIter.next(); - batchIter.remove(); - - assertEquals("1st column before commit", 1, map.iterator().next().name().toByteBuffer().getInt(0)); - - batchIter.commit(); - - assertEquals("1st column after commit", 3, map.iterator().next().name().toByteBuffer().getInt(0)); - } - - @Test - public void testBatchRemoveCopy() - { - // Test delete some random columns and check the result - CellNameType type = new SimpleDenseCellNameType(Int32Type.instance); - ColumnFamily map = ArrayBackedSortedColumns.factory.create(metadata(), false); - int n = 127; - int[] values = new int[n]; - for (int i = 0; i < n; i++) - values[i] = i; - Set toRemove = Sets.newHashSet(3, 12, 13, 15, 58, 103, 112); - - for (int value : values) - map.addColumn(new BufferCell(type.makeCellName(value))); - - BatchRemoveIterator batchIter = map.batchRemoveIterator(); - while (batchIter.hasNext()) - if (toRemove.contains(batchIter.next().name().toByteBuffer().getInt(0))) - batchIter.remove(); - - batchIter.commit(); - - int expected = 0; - while (toRemove.contains(expected)) - expected++; - - for (Cell column : map) - { - assertEquals(expected, column.name().toByteBuffer().getInt(0)); - expected++; - while (toRemove.contains(expected)) - expected++; - } - assertEquals(expected, n); - } -} diff --git a/test/unit/org/apache/cassandra/db/BatchlogManagerTest.java b/test/unit/org/apache/cassandra/db/BatchlogManagerTest.java index 70d1d0ce75..c81fc4f09b 100644 --- a/test/unit/org/apache/cassandra/db/BatchlogManagerTest.java +++ b/test/unit/org/apache/cassandra/db/BatchlogManagerTest.java @@ -19,10 +19,15 @@ package org.apache.cassandra.db; import java.net.InetAddress; import java.util.Collections; -import java.util.List; -import java.util.concurrent.ExecutionException; +import java.util.Iterator; + +import org.apache.cassandra.config.CFMetaData; +import org.apache.cassandra.db.rows.Row; +import org.apache.cassandra.db.partitions.ArrayBackedPartition; +import org.apache.cassandra.db.partitions.PartitionUpdate; +import org.apache.cassandra.utils.ByteBufferUtil; +import org.apache.cassandra.utils.FBUtilities; -import com.google.common.collect.Lists; import org.junit.BeforeClass; import org.junit.Before; import org.junit.Test; @@ -33,8 +38,7 @@ import org.apache.cassandra.config.DatabaseDescriptor; import org.apache.cassandra.config.KSMetaData; import org.apache.cassandra.cql3.QueryProcessor; import org.apache.cassandra.cql3.UntypedResultSet; -import org.apache.cassandra.db.composites.CellNameType; -import org.apache.cassandra.db.commitlog.ReplayPosition; +import org.apache.cassandra.db.marshal.BytesType; import org.apache.cassandra.exceptions.ConfigurationException; import org.apache.cassandra.locator.TokenMetadata; import org.apache.cassandra.locator.SimpleStrategy; @@ -61,7 +65,7 @@ public class BatchlogManagerTest SchemaLoader.createKeyspace(KEYSPACE1, SimpleStrategy.class, KSMetaData.optsWithRF(1), - SchemaLoader.standardCFMD(KEYSPACE1, CF_STANDARD1), + SchemaLoader.standardCFMD(KEYSPACE1, CF_STANDARD1, 1, BytesType.instance), SchemaLoader.standardCFMD(KEYSPACE1, CF_STANDARD2), SchemaLoader.standardCFMD(KEYSPACE1, CF_STANDARD3)); } @@ -75,29 +79,59 @@ public class BatchlogManagerTest metadata.updateHostId(UUIDGen.getTimeUUID(), localhost); } + @Test + public void testDelete() + { + ColumnFamilyStore cfs = Keyspace.open(KEYSPACE1).getColumnFamilyStore(CF_STANDARD1); + CFMetaData cfm = cfs.metadata; + new RowUpdateBuilder(cfm, FBUtilities.timestampMicros(), ByteBufferUtil.bytes("1234")) + .clustering("c") + .add("val", "val" + 1234) + .build() + .applyUnsafe(); + + DecoratedKey dk = StorageService.getPartitioner().decorateKey(ByteBufferUtil.bytes("1234")); + ArrayBackedPartition results = Util.getOnlyPartitionUnfiltered(Util.cmd(cfs, dk).build()); + Iterator iter = results.iterator(); + assert iter.hasNext(); + + Mutation mutation = new Mutation(KEYSPACE1, dk); + mutation.add(PartitionUpdate.fullPartitionDelete(cfm, + mutation.key(), + FBUtilities.timestampMicros(), + FBUtilities.nowInSeconds())); + mutation.applyUnsafe(); + + Util.assertEmpty(Util.cmd(cfs, dk).build()); + } + + // TODO: Fix. Currently endlessly looping on BatchLogManager.replayAllFailedBatches @Test public void testReplay() throws Exception { long initialAllBatches = BatchlogManager.instance.countAllBatches(); long initialReplayedBatches = BatchlogManager.instance.getTotalBatchesReplayed(); + CFMetaData cfm = Keyspace.open(KEYSPACE1).getColumnFamilyStore(CF_STANDARD1).metadata; + // Generate 1000 mutations and put them all into the batchlog. // Half (500) ready to be replayed, half not. - CellNameType comparator = Keyspace.open(KEYSPACE1).getColumnFamilyStore("Standard1").metadata.comparator; for (int i = 0; i < 1000; i++) { - Mutation mutation = new Mutation(KEYSPACE1, bytes(i)); - mutation.add("Standard1", comparator.makeCellName(bytes(i)), bytes(i), System.currentTimeMillis()); + Mutation m = new RowUpdateBuilder(cfm, FBUtilities.timestampMicros(), bytes(i)) + .clustering("name" + i) + .add("val", "val" + i) + .build(); long timestamp = i < 500 ? (System.currentTimeMillis() - DatabaseDescriptor.getWriteRpcTimeout() * 2) * 1000 : Long.MAX_VALUE; - BatchlogManager.getBatchlogMutationFor(Collections.singleton(mutation), + Mutation m2 = BatchlogManager.getBatchlogMutationFor(Collections.singleton(m), UUIDGen.getTimeUUID(), MessagingService.current_version, - timestamp) - .applyUnsafe(); + timestamp); + m2.applyUnsafe(); } // Flush the batchlog to disk (see CASSANDRA-6822). @@ -119,8 +153,8 @@ public class BatchlogManagerTest if (i < 500) { assertEquals(bytes(i), result.one().getBytes("key")); - assertEquals(bytes(i), result.one().getBytes("column1")); - assertEquals(bytes(i), result.one().getBytes("value")); + assertEquals("name" + i, result.one().getString("name")); + assertEquals("val" + i, result.one().getString("val")); } else { @@ -133,6 +167,7 @@ public class BatchlogManagerTest assertEquals(500, result.one().getLong("count")); } + /* @Test public void testTruncatedReplay() throws InterruptedException, ExecutionException { @@ -200,4 +235,5 @@ public class BatchlogManagerTest assertEquals(bytes(i), result.one().getBytes("value")); } } + */ } diff --git a/test/unit/org/apache/cassandra/db/CellTest.java b/test/unit/org/apache/cassandra/db/CellTest.java index 493dbbf7dc..a7e4bf745a 100644 --- a/test/unit/org/apache/cassandra/db/CellTest.java +++ b/test/unit/org/apache/cassandra/db/CellTest.java @@ -23,20 +23,36 @@ package org.apache.cassandra.db; import java.nio.ByteBuffer; +import junit.framework.Assert; +import org.junit.BeforeClass; import org.junit.Test; -import junit.framework.Assert; -import org.apache.cassandra.Util; +import org.apache.cassandra.config.CFMetaData; +import org.apache.cassandra.config.ColumnDefinition; +import org.apache.cassandra.config.KSMetaData; +import org.apache.cassandra.db.rows.*; +import org.apache.cassandra.exceptions.ConfigurationException; +import org.apache.cassandra.locator.SimpleStrategy; +import org.apache.cassandra.SchemaLoader; import org.apache.cassandra.utils.ByteBufferUtil; -import org.apache.cassandra.utils.concurrent.OpOrder; -import org.apache.cassandra.utils.memory.NativeAllocator; -import org.apache.cassandra.utils.memory.NativePool; +import org.apache.cassandra.utils.FBUtilities; public class CellTest { + private static final String KEYSPACE1 = "CellTest"; + private static final String CF_STANDARD1 = "Standard1"; - private static final OpOrder order = new OpOrder(); - private static NativeAllocator allocator = new NativePool(Integer.MAX_VALUE, Integer.MAX_VALUE, 1f, null).newAllocator(); + private CFMetaData cfm = SchemaLoader.standardCFMD(KEYSPACE1, CF_STANDARD1); + + @BeforeClass + public static void defineSchema() throws ConfigurationException + { + SchemaLoader.prepareServer(); + SchemaLoader.createKeyspace(KEYSPACE1, + SimpleStrategy.class, + KSMetaData.optsWithRF(1), + SchemaLoader.standardCFMD(KEYSPACE1, CF_STANDARD1)); + } @Test public void testConflictingTypeEquality() @@ -49,12 +65,12 @@ public class CellTest // don't test equality for both sides native, as this is based on CellName resolution if (lhs && rhs) continue; - Cell a = expiring("a", "a", 1, 1, lhs); - Cell b = regular("a", "a", 1, rhs); + Cell a = buildCell(cfm, "a", "a", 1, 1); + Cell b = buildCell(cfm, "a", "a", 1, 0); Assert.assertNotSame(a, b); Assert.assertNotSame(b, a); - a = deleted("a", 1, 1, lhs); - b = regular("a", ByteBufferUtil.bytes(1), 1, rhs); + + a = deleted(cfm, "a", "a", 1, 1); Assert.assertNotSame(a, b); Assert.assertNotSame(b, a); } @@ -71,7 +87,6 @@ public class CellTest Assert.assertEquals(-1, testExpiring("a", "a", 2, 1, null, null, 1L, null)); Assert.assertEquals(-1, testExpiring("a", "a", 2, 1, null, "b", 1L, 2)); - // newer TTL Assert.assertEquals(-1, testExpiring("a", "a", 1, 2, null, null, null, 1)); Assert.assertEquals(1, testExpiring("a", "a", 1, 2, null, "b", null, 1)); @@ -90,52 +105,65 @@ public class CellTest t2 = t1; if (et2 == null) et2 = et1; - int result = testExpiring(n1, v1, t1, et1, false, n2, v2, t2, et2, false); - Assert.assertEquals(result, testExpiring(n1, v1, t1, et1, false, n2, v2, t2, et2, true)); - Assert.assertEquals(result, testExpiring(n1, v1, t1, et1, true, n2, v2, t2, et2, false)); - Assert.assertEquals(result, testExpiring(n1, v1, t1, et1, true, n2, v2, t2, et2, true)); - return result; + Cell c1 = buildCell(cfm, n1, v1, t1, et1); + Cell c2 = buildCell(cfm, n2, v2, t2, et2); + + int now = FBUtilities.nowInSeconds(); + if (Cells.reconcile(c1, c2, now) == c1) + return Cells.reconcile(c2, c1, now) == c1 ? -1 : 0; + return Cells.reconcile(c2, c1, now) == c2 ? 1 : 0; } - private int testExpiring(String n1, String v1, long t1, int et1, boolean native1, String n2, String v2, long t2, int et2, boolean native2) + private Cell buildCell(CFMetaData cfm, String columnName, String value, long timestamp, int ttl) { - Cell c1 = expiring(n1, v1, t1, et1, native1); - Cell c2 = expiring(n2, v2, t2, et2, native2); - return reconcile(c1, c2); + ColumnDefinition cdef = cfm.getColumnDefinition(ByteBufferUtil.bytes(columnName)); + LivenessInfo info = SimpleLivenessInfo.forUpdate(timestamp, ttl, FBUtilities.nowInSeconds(), cfm); + return new TestCell(cdef, ByteBufferUtil.bytes(value), info); } - int reconcile(Cell c1, Cell c2) + private Cell deleted(CFMetaData cfm, String columnName, String value, int localDeletionTime, long timestamp) { - if (c1.reconcile(c2) == c1) - return c2.reconcile(c1) == c1 ? -1 : 0; - return c2.reconcile(c1) == c2 ? 1 : 0; + ColumnDefinition cdef = cfm.getColumnDefinition(ByteBufferUtil.bytes(columnName)); + LivenessInfo info = SimpleLivenessInfo.forDeletion(timestamp, localDeletionTime); + return new TestCell(cdef, ByteBufferUtil.bytes(value), info); } - private Cell expiring(String name, String value, long timestamp, int expirationTime, boolean nativeCell) + public static class TestCell extends AbstractCell { - ExpiringCell cell = new BufferExpiringCell(Util.cellname(name), ByteBufferUtil.bytes(value), timestamp, 1, expirationTime); - if (nativeCell) - cell = new NativeExpiringCell(allocator, order.getCurrent(), cell); - return cell; - } + private final ColumnDefinition column; + private final ByteBuffer value; + private final LivenessInfo info; - private Cell regular(String name, ByteBuffer value, long timestamp, boolean nativeCell) - { - Cell cell = new BufferCell(Util.cellname(name), value, timestamp); - if (nativeCell) - cell = new NativeCell(allocator, order.getCurrent(), cell); - return cell; - } - private Cell regular(String name, String value, long timestamp, boolean nativeCell) - { - return regular(name, ByteBufferUtil.bytes(value), timestamp, nativeCell); - } + public TestCell(ColumnDefinition column, ByteBuffer value, LivenessInfo info) + { + this.column = column; + this.value = value; + this.info = info.takeAlias(); + } - private Cell deleted(String name, int localDeletionTime, long timestamp, boolean nativeCell) - { - DeletedCell cell = new BufferDeletedCell(Util.cellname(name), localDeletionTime, timestamp); - if (nativeCell) - cell = new NativeDeletedCell(allocator, order.getCurrent(), cell); - return cell; + public ColumnDefinition column() + { + return column; + } + + public boolean isCounterCell() + { + return false; + } + + public ByteBuffer value() + { + return value; + } + + public LivenessInfo livenessInfo() + { + return info; + } + + public CellPath path() + { + return null; + } } } diff --git a/test/unit/org/apache/cassandra/db/CleanupTest.java b/test/unit/org/apache/cassandra/db/CleanupTest.java index 81f7d41a76..d1214cbd31 100644 --- a/test/unit/org/apache/cassandra/db/CleanupTest.java +++ b/test/unit/org/apache/cassandra/db/CleanupTest.java @@ -22,12 +22,14 @@ import java.io.IOException; import java.net.InetAddress; import java.net.UnknownHostException; import java.nio.ByteBuffer; -import java.util.Arrays; import java.util.LinkedList; import java.util.List; import java.util.concurrent.ExecutionException; import java.util.concurrent.TimeUnit; +import org.apache.cassandra.config.ColumnDefinition; +import org.apache.cassandra.db.filter.RowFilter; + import org.junit.BeforeClass; import org.junit.Test; @@ -35,12 +37,9 @@ import org.apache.cassandra.SchemaLoader; import org.apache.cassandra.Util; import org.apache.cassandra.config.KSMetaData; import org.apache.cassandra.cql3.Operator; -import org.apache.cassandra.db.filter.IDiskAtomFilter; -import org.apache.cassandra.db.columniterator.IdentityQueryFilter; import org.apache.cassandra.db.compaction.CompactionManager; import org.apache.cassandra.db.index.SecondaryIndex; import org.apache.cassandra.dht.ByteOrderedPartitioner.BytesToken; -import org.apache.cassandra.dht.Range; import org.apache.cassandra.exceptions.ConfigurationException; import org.apache.cassandra.io.sstable.format.SSTableReader; import org.apache.cassandra.locator.SimpleStrategy; @@ -54,8 +53,8 @@ public class CleanupTest { public static final int LOOPS = 200; public static final String KEYSPACE1 = "CleanupTest1"; - public static final String CF1 = "Indexed1"; - public static final String CF2 = "Standard1"; + public static final String CF_INDEXED1 = "Indexed1"; + public static final String CF_STANDARD1 = "Standard1"; public static final ByteBuffer COLUMN = ByteBufferUtil.bytes("birthdate"); public static final ByteBuffer VALUE = ByteBuffer.allocate(8); static @@ -71,28 +70,29 @@ public class CleanupTest SchemaLoader.createKeyspace(KEYSPACE1, SimpleStrategy.class, KSMetaData.optsWithRF(1), - SchemaLoader.standardCFMD(KEYSPACE1, CF2), - SchemaLoader.indexCFMD(KEYSPACE1, CF1, true)); + SchemaLoader.standardCFMD(KEYSPACE1, CF_STANDARD1), + SchemaLoader.compositeIndexCFMD(KEYSPACE1, CF_INDEXED1, true)); } + /* @Test public void testCleanup() throws ExecutionException, InterruptedException { StorageService.instance.getTokenMetadata().clearUnsafe(); Keyspace keyspace = Keyspace.open(KEYSPACE1); - ColumnFamilyStore cfs = keyspace.getColumnFamilyStore(CF2); + ColumnFamilyStore cfs = keyspace.getColumnFamilyStore(CF_STANDARD1); - List rows; + UnfilteredPartitionIterator iter; // insert data and verify we get it back w/ range query - fillCF(cfs, LOOPS); + fillCF(cfs, "val", LOOPS); // record max timestamps of the sstables pre-cleanup List expectedMaxTimestamps = getMaxTimestampList(cfs); - rows = Util.getRangeSlice(cfs); - assertEquals(LOOPS, rows.size()); + iter = Util.getRangeSlice(cfs); + assertEquals(LOOPS, Iterators.size(iter)); // with one token in the ring, owned by the local node, cleanup should be a no-op CompactionManager.instance.performCleanup(cfs); @@ -101,35 +101,31 @@ public class CleanupTest assert expectedMaxTimestamps.equals(getMaxTimestampList(cfs)); // check data is still there - rows = Util.getRangeSlice(cfs); - assertEquals(LOOPS, rows.size()); + iter = Util.getRangeSlice(cfs); + assertEquals(LOOPS, Iterators.size(iter)); } + */ @Test public void testCleanupWithIndexes() throws IOException, ExecutionException, InterruptedException { Keyspace keyspace = Keyspace.open(KEYSPACE1); - ColumnFamilyStore cfs = keyspace.getColumnFamilyStore(CF1); + ColumnFamilyStore cfs = keyspace.getColumnFamilyStore(CF_INDEXED1); - List rows; // insert data and verify we get it back w/ range query - fillCF(cfs, LOOPS); - rows = Util.getRangeSlice(cfs); - assertEquals(LOOPS, rows.size()); + fillCF(cfs, "birthdate", LOOPS); + assertEquals(LOOPS, Util.getAll(Util.cmd(cfs).build()).size()); - SecondaryIndex index = cfs.indexManager.getIndexForColumn(COLUMN); + SecondaryIndex index = cfs.indexManager.getIndexForColumn(cfs.metadata.getColumnDefinition(COLUMN)); long start = System.nanoTime(); while (!index.isIndexBuilt(COLUMN) && System.nanoTime() - start < TimeUnit.SECONDS.toNanos(10)) Thread.sleep(10); - // verify we get it back w/ index query too - IndexExpression expr = new IndexExpression(COLUMN, Operator.EQ, VALUE); - List clause = Arrays.asList(expr); - IDiskAtomFilter filter = new IdentityQueryFilter(); - Range range = Util.range("", ""); - rows = keyspace.getColumnFamilyStore(CF1).search(range, clause, filter, Integer.MAX_VALUE); - assertEquals(LOOPS, rows.size()); + ColumnDefinition cdef = cfs.metadata.getColumnDefinition(COLUMN); + RowFilter cf = RowFilter.create(); + cf.add(cdef, Operator.EQ, VALUE); + assertEquals(LOOPS, Util.getAll(Util.cmd(cfs).filterOn("birthdate", Operator.EQ, VALUE).build()).size()); // we don't allow cleanup when the local host has no range to avoid wipping up all data when a node has not join the ring. // So to make sure cleanup erase everything here, we give the localhost the tiniest possible range. @@ -143,15 +139,13 @@ public class CleanupTest CompactionManager.instance.performCleanup(cfs); // row data should be gone - rows = Util.getRangeSlice(cfs); - assertEquals(0, rows.size()); + assertEquals(0, Util.getAll(Util.cmd(cfs).build()).size()); // not only should it be gone but there should be no data on disk, not even tombstones assert cfs.getSSTables().isEmpty(); // 2ary indexes should result in no results, too (although tombstones won't be gone until compacted) - rows = cfs.search(range, clause, filter, Integer.MAX_VALUE); - assertEquals(0, rows.size()); + assertEquals(0, Util.getAll(Util.cmd(cfs).filterOn("birthdate", Operator.EQ, VALUE).build()).size()); } @Test @@ -160,16 +154,12 @@ public class CleanupTest StorageService.instance.getTokenMetadata().clearUnsafe(); Keyspace keyspace = Keyspace.open(KEYSPACE1); - ColumnFamilyStore cfs = keyspace.getColumnFamilyStore(CF2); - - List rows; + ColumnFamilyStore cfs = keyspace.getColumnFamilyStore(CF_STANDARD1); // insert data and verify we get it back w/ range query - fillCF(cfs, LOOPS); + fillCF(cfs, "val", LOOPS); - rows = Util.getRangeSlice(cfs); - - assertEquals(LOOPS, rows.size()); + assertEquals(LOOPS, Util.getAll(Util.cmd(cfs).build()).size()); TokenMetadata tmd = StorageService.instance.getTokenMetadata(); byte[] tk1 = new byte[1], tk2 = new byte[1]; @@ -179,11 +169,10 @@ public class CleanupTest tmd.updateNormalToken(new BytesToken(tk2), InetAddress.getByName("127.0.0.2")); CompactionManager.instance.performCleanup(cfs); - rows = Util.getRangeSlice(cfs); - assertEquals(0, rows.size()); + assertEquals(0, Util.getAll(Util.cmd(cfs).build()).size()); } - protected void fillCF(ColumnFamilyStore cfs, int rowsPerSSTable) + protected void fillCF(ColumnFamilyStore cfs, String colName, int rowsPerSSTable) { CompactionManager.instance.disableAutoCompaction(); @@ -191,10 +180,11 @@ public class CleanupTest { String key = String.valueOf(i); // create a row and update the birthdate value, test that the index query fetches the new version - Mutation rm; - rm = new Mutation(KEYSPACE1, ByteBufferUtil.bytes(key)); - rm.add(cfs.name, Util.cellname(COLUMN), VALUE, System.currentTimeMillis()); - rm.applyUnsafe(); + new RowUpdateBuilder(cfs.metadata, System.currentTimeMillis(), ByteBufferUtil.bytes(key)) + .clustering(COLUMN) + .add(colName, VALUE) + .build() + .applyUnsafe(); } cfs.forceBlockingFlush(); diff --git a/test/unit/org/apache/cassandra/db/CollationControllerTest.java b/test/unit/org/apache/cassandra/db/CollationControllerTest.java deleted file mode 100644 index c227816c2e..0000000000 --- a/test/unit/org/apache/cassandra/db/CollationControllerTest.java +++ /dev/null @@ -1,138 +0,0 @@ -/* -* Licensed to the Apache Software Foundation (ASF) under one -* or more contributor license agreements. See the NOTICE file -* distributed with this work for additional information -* regarding copyright ownership. The ASF licenses this file -* to you under the Apache License, Version 2.0 (the -* "License"); you may not use this file except in compliance -* with the License. You may obtain a copy of the License at -* -* http://www.apache.org/licenses/LICENSE-2.0 -* -* Unless required by applicable law or agreed to in writing, -* software distributed under the License is distributed on an -* "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -* KIND, either express or implied. See the License for the -* specific language governing permissions and limitations -* under the License. -*/ -package org.apache.cassandra.db; - -import org.junit.BeforeClass; -import org.junit.Test; - -import org.apache.cassandra.SchemaLoader; -import org.apache.cassandra.Util; -import org.apache.cassandra.config.KSMetaData; -import org.apache.cassandra.db.composites.CellName; -import org.apache.cassandra.db.filter.QueryFilter; -import org.apache.cassandra.exceptions.ConfigurationException; -import org.apache.cassandra.locator.SimpleStrategy; -import org.apache.cassandra.utils.ByteBufferUtil; -import org.apache.cassandra.utils.FBUtilities; - -import static org.junit.Assert.assertEquals; - -public class CollationControllerTest -{ - private static final String KEYSPACE1 = "CollationControllerTest"; - private static final String CF = "Standard1"; - private static final String CFGCGRACE = "StandardGCGS0"; - - @BeforeClass - public static void defineSchema() throws ConfigurationException - { - SchemaLoader.prepareServer(); - SchemaLoader.createKeyspace(KEYSPACE1, - SimpleStrategy.class, - KSMetaData.optsWithRF(1), - SchemaLoader.standardCFMD(KEYSPACE1, CF), - SchemaLoader.standardCFMD(KEYSPACE1, CFGCGRACE).gcGraceSeconds(0)); - } - - @Test - public void getTopLevelColumnsSkipsSSTablesModifiedBeforeRowDelete() - { - Keyspace keyspace = Keyspace.open(KEYSPACE1); - ColumnFamilyStore cfs = keyspace.getColumnFamilyStore(CF); - Mutation rm; - DecoratedKey dk = Util.dk("key1"); - - // add data - rm = new Mutation(keyspace.getName(), dk.getKey()); - rm.add(cfs.name, Util.cellname("Column1"), ByteBufferUtil.bytes("asdf"), 0); - rm.applyUnsafe(); - cfs.forceBlockingFlush(); - - // remove - rm = new Mutation(keyspace.getName(), dk.getKey()); - rm.delete(cfs.name, 10); - rm.applyUnsafe(); - - // add another mutation because sstable maxtimestamp isn't set - // correctly during flush if the most recent mutation is a row delete - rm = new Mutation(keyspace.getName(), Util.dk("key2").getKey()); - rm.add(cfs.name, Util.cellname("Column1"), ByteBufferUtil.bytes("zxcv"), 20); - rm.applyUnsafe(); - - cfs.forceBlockingFlush(); - - // add yet one more mutation - rm = new Mutation(keyspace.getName(), dk.getKey()); - rm.add(cfs.name, Util.cellname("Column1"), ByteBufferUtil.bytes("foobar"), 30); - rm.applyUnsafe(); - cfs.forceBlockingFlush(); - - // A NamesQueryFilter goes down one code path (through collectTimeOrderedData()) - // It should only iterate the last flushed sstable, since it probably contains the most recent value for Column1 - QueryFilter filter = Util.namesQueryFilter(cfs, dk, "Column1"); - CollationController controller = new CollationController(cfs, filter, Integer.MIN_VALUE); - controller.getTopLevelColumns(true); - assertEquals(1, controller.getSstablesIterated()); - - // SliceQueryFilter goes down another path (through collectAllData()) - // We will read "only" the last sstable in that case, but because the 2nd sstable has a tombstone that is more - // recent than the maxTimestamp of the very first sstable we flushed, we should only read the 2 first sstables. - filter = QueryFilter.getIdentityFilter(dk, cfs.name, System.currentTimeMillis()); - controller = new CollationController(cfs, filter, Integer.MIN_VALUE); - controller.getTopLevelColumns(true); - assertEquals(2, controller.getSstablesIterated()); - } - - @Test - public void ensureTombstonesAppliedAfterGCGS() - { - Keyspace keyspace = Keyspace.open(KEYSPACE1); - ColumnFamilyStore cfs = keyspace.getColumnFamilyStore(CFGCGRACE); - cfs.disableAutoCompaction(); - - Mutation rm; - DecoratedKey dk = Util.dk("key1"); - CellName cellName = Util.cellname("Column1"); - - // add data - rm = new Mutation(keyspace.getName(), dk.getKey()); - rm.add(cfs.name, cellName, ByteBufferUtil.bytes("asdf"), 0); - rm.applyUnsafe(); - cfs.forceBlockingFlush(); - - // remove - rm = new Mutation(keyspace.getName(), dk.getKey()); - rm.delete(cfs.name, cellName, 0); - rm.applyUnsafe(); - cfs.forceBlockingFlush(); - - // use "realistic" query times since we'll compare these numbers to the local deletion time of the tombstone - QueryFilter filter; - long queryAt = System.currentTimeMillis() + 1000; - int gcBefore = cfs.gcBefore(queryAt); - - filter = QueryFilter.getNamesFilter(dk, cfs.name, FBUtilities.singleton(cellName, cfs.getComparator()), queryAt); - CollationController controller = new CollationController(cfs, filter, gcBefore); - assert ColumnFamilyStore.removeDeleted(controller.getTopLevelColumns(true), gcBefore) == null; - - filter = QueryFilter.getIdentityFilter(dk, cfs.name, queryAt); - controller = new CollationController(cfs, filter, gcBefore); - assert ColumnFamilyStore.removeDeleted(controller.getTopLevelColumns(true), gcBefore) == null; - } -} diff --git a/test/unit/org/apache/cassandra/db/ColumnFamilyMetricTest.java b/test/unit/org/apache/cassandra/db/ColumnFamilyMetricTest.java index b644264a7b..9c642c77ef 100644 --- a/test/unit/org/apache/cassandra/db/ColumnFamilyMetricTest.java +++ b/test/unit/org/apache/cassandra/db/ColumnFamilyMetricTest.java @@ -21,6 +21,7 @@ import java.nio.ByteBuffer; import java.util.Collection; import java.util.function.Supplier; +import org.apache.cassandra.utils.FBUtilities; import org.junit.BeforeClass; import org.junit.Test; import org.apache.cassandra.SchemaLoader; @@ -31,7 +32,6 @@ import org.apache.cassandra.locator.SimpleStrategy; import org.apache.cassandra.utils.ByteBufferUtil; import static org.junit.Assert.assertEquals; -import static org.apache.cassandra.Util.cellname; public class ColumnFamilyMetricTest { @@ -49,23 +49,24 @@ public class ColumnFamilyMetricTest public void testSizeMetric() { Keyspace keyspace = Keyspace.open("Keyspace1"); - ColumnFamilyStore store = keyspace.getColumnFamilyStore("Standard2"); - store.disableAutoCompaction(); + ColumnFamilyStore cfs = keyspace.getColumnFamilyStore("Standard2"); + cfs.disableAutoCompaction(); - store.truncateBlocking(); + cfs.truncateBlocking(); - assertEquals(0, store.metric.liveDiskSpaceUsed.getCount()); - assertEquals(0, store.metric.totalDiskSpaceUsed.getCount()); + assertEquals(0, cfs.metric.liveDiskSpaceUsed.getCount()); + assertEquals(0, cfs.metric.totalDiskSpaceUsed.getCount()); for (int j = 0; j < 10; j++) { - ByteBuffer key = ByteBufferUtil.bytes(String.valueOf(j)); - Mutation rm = new Mutation("Keyspace1", key); - rm.add("Standard2", cellname("0"), ByteBufferUtil.EMPTY_BYTE_BUFFER, j); - rm.apply(); + new RowUpdateBuilder(cfs.metadata, FBUtilities.timestampMicros(), String.valueOf(j)) + .clustering("0") + .add("val", ByteBufferUtil.EMPTY_BYTE_BUFFER) + .build() + .applyUnsafe(); } - store.forceBlockingFlush(); - Collection sstables = store.getSSTables(); + cfs.forceBlockingFlush(); + Collection sstables = cfs.getSSTables(); long size = 0; for (SSTableReader reader : sstables) { @@ -73,15 +74,15 @@ public class ColumnFamilyMetricTest } // size metrics should show the sum of all SSTable sizes - assertEquals(size, store.metric.liveDiskSpaceUsed.getCount()); - assertEquals(size, store.metric.totalDiskSpaceUsed.getCount()); + assertEquals(size, cfs.metric.liveDiskSpaceUsed.getCount()); + assertEquals(size, cfs.metric.totalDiskSpaceUsed.getCount()); - store.truncateBlocking(); + cfs.truncateBlocking(); // after truncate, size metrics should be down to 0 - Util.spinAssertEquals(0L, () -> store.metric.liveDiskSpaceUsed.getCount(), 30); - Util.spinAssertEquals(0L, () -> store.metric.totalDiskSpaceUsed.getCount(), 30); + Util.spinAssertEquals(0L, () -> cfs.metric.liveDiskSpaceUsed.getCount(), 30); + Util.spinAssertEquals(0L, () -> cfs.metric.totalDiskSpaceUsed.getCount(), 30); - store.enableAutoCompaction(); + cfs.enableAutoCompaction(); } } diff --git a/test/unit/org/apache/cassandra/db/ColumnFamilyStoreTest.java b/test/unit/org/apache/cassandra/db/ColumnFamilyStoreTest.java index c85b2e0d62..cf17df7f33 100644 --- a/test/unit/org/apache/cassandra/db/ColumnFamilyStoreTest.java +++ b/test/unit/org/apache/cassandra/db/ColumnFamilyStoreTest.java @@ -21,119 +21,43 @@ package org.apache.cassandra.db; import java.io.File; import java.io.IOException; import java.nio.ByteBuffer; -import java.nio.charset.CharacterCodingException; -import java.util.Arrays; -import java.util.Collection; -import java.util.Collections; -import java.util.HashMap; -import java.util.HashSet; -import java.util.Iterator; -import java.util.LinkedList; -import java.util.List; -import java.util.Map; -import java.util.Random; -import java.util.Set; -import java.util.SortedSet; -import java.util.TreeSet; -import java.util.UUID; -import java.util.concurrent.ExecutionException; -import java.util.concurrent.Future; -import java.util.concurrent.TimeUnit; +import java.util.*; -import com.google.common.base.Function; -import com.google.common.collect.Iterables; -import com.google.common.collect.Sets; - -import org.apache.cassandra.db.index.PerRowSecondaryIndexTest; -import org.apache.cassandra.io.sstable.*; -import org.apache.cassandra.io.sstable.format.SSTableReader; -import org.apache.cassandra.io.sstable.format.SSTableWriter; -import org.apache.commons.lang3.ArrayUtils; -import org.apache.commons.lang3.StringUtils; +import org.junit.Before; import org.junit.BeforeClass; import org.junit.Test; import org.junit.runner.RunWith; -import org.apache.cassandra.OrderedJUnit4ClassRunner; -import org.apache.cassandra.SchemaLoader; -import org.apache.cassandra.Util; +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertTrue; + +import com.google.common.collect.Iterators; +import org.apache.cassandra.*; import org.apache.cassandra.config.*; import org.apache.cassandra.cql3.Operator; -import org.apache.cassandra.db.columniterator.IdentityQueryFilter; -import org.apache.cassandra.db.composites.CellName; -import org.apache.cassandra.db.composites.CellNameType; -import org.apache.cassandra.db.composites.CellNames; -import org.apache.cassandra.db.composites.Composites; -import org.apache.cassandra.db.filter.ColumnSlice; -import org.apache.cassandra.db.filter.IDiskAtomFilter; -import org.apache.cassandra.db.filter.NamesQueryFilter; -import org.apache.cassandra.db.filter.QueryFilter; -import org.apache.cassandra.db.filter.SliceQueryFilter; -import org.apache.cassandra.db.index.SecondaryIndex; -import org.apache.cassandra.db.marshal.IntegerType; -import org.apache.cassandra.db.marshal.LexicalUUIDType; -import org.apache.cassandra.db.marshal.LongType; -import org.apache.cassandra.db.marshal.UTF8Type; -import org.apache.cassandra.dht.Bounds; -import org.apache.cassandra.dht.ExcludingBounds; -import org.apache.cassandra.dht.IPartitioner; -import org.apache.cassandra.dht.IncludingExcludingBounds; -import org.apache.cassandra.dht.Range; +import org.apache.cassandra.db.rows.*; +import org.apache.cassandra.db.partitions.*; +import org.apache.cassandra.db.marshal.*; +import org.apache.cassandra.db.partitions.PartitionIterator; import org.apache.cassandra.exceptions.ConfigurationException; -import org.apache.cassandra.io.sstable.metadata.MetadataCollector; -import org.apache.cassandra.io.util.FileUtils; +import org.apache.cassandra.io.sstable.Component; +import org.apache.cassandra.io.sstable.Descriptor; +import org.apache.cassandra.io.sstable.format.SSTableReader; import org.apache.cassandra.locator.SimpleStrategy; import org.apache.cassandra.metrics.ClearableHistogram; -import org.apache.cassandra.service.ActiveRepairService; -import org.apache.cassandra.service.StorageService; -import org.apache.cassandra.thrift.SlicePredicate; -import org.apache.cassandra.thrift.SliceRange; -import org.apache.cassandra.thrift.ThriftValidation; import org.apache.cassandra.utils.ByteBufferUtil; import org.apache.cassandra.utils.FBUtilities; -import org.apache.cassandra.utils.Pair; import org.apache.cassandra.utils.WrappedRunnable; -import org.apache.thrift.TException; - -import static org.apache.cassandra.Util.cellname; -import static org.apache.cassandra.Util.column; -import static org.apache.cassandra.Util.dk; -import static org.apache.cassandra.Util.rp; -import static org.apache.cassandra.utils.ByteBufferUtil.bytes; -import static org.junit.Assert.assertEquals; -import static org.junit.Assert.assertFalse; -import static org.junit.Assert.assertNull; -import static org.junit.Assert.assertSame; -import static org.junit.Assert.assertTrue; @RunWith(OrderedJUnit4ClassRunner.class) public class ColumnFamilyStoreTest { - static byte[] bytes1, bytes2; public static final String KEYSPACE1 = "ColumnFamilyStoreTest1"; public static final String KEYSPACE2 = "ColumnFamilyStoreTest2"; - public static final String KEYSPACE3 = "ColumnFamilyStoreTest3"; - public static final String KEYSPACE4 = "PerRowSecondaryIndex"; public static final String CF_STANDARD1 = "Standard1"; public static final String CF_STANDARD2 = "Standard2"; - public static final String CF_STANDARD3 = "Standard3"; - public static final String CF_STANDARD4 = "Standard4"; - public static final String CF_STANDARD5 = "Standard5"; - public static final String CF_STANDARDINT = "StandardInteger1"; public static final String CF_SUPER1 = "Super1"; public static final String CF_SUPER6 = "Super6"; - public static final String CF_INDEX1 = "Indexed1"; - public static final String CF_INDEX2 = "Indexed2"; - public static final String CF_INDEX3 = "Indexed3"; - - static - { - Random random = new Random(); - bytes1 = new byte[1024]; - bytes2 = new byte[128]; - random.nextBytes(bytes1); - random.nextBytes(bytes2); - } @BeforeClass public static void defineSchema() throws ConfigurationException @@ -143,30 +67,24 @@ public class ColumnFamilyStoreTest SimpleStrategy.class, KSMetaData.optsWithRF(1), SchemaLoader.standardCFMD(KEYSPACE1, CF_STANDARD1), - SchemaLoader.standardCFMD(KEYSPACE1, CF_STANDARD2), - SchemaLoader.standardCFMD(KEYSPACE1, CF_STANDARD3), - SchemaLoader.standardCFMD(KEYSPACE1, CF_STANDARD4), - SchemaLoader.standardCFMD(KEYSPACE1, CF_STANDARD5), - SchemaLoader.indexCFMD(KEYSPACE1, CF_INDEX1, true), - SchemaLoader.indexCFMD(KEYSPACE1, CF_INDEX2, false), - SchemaLoader.superCFMD(KEYSPACE1, CF_SUPER1, LongType.instance), - SchemaLoader.superCFMD(KEYSPACE1, CF_SUPER6, LexicalUUIDType.instance, UTF8Type.instance), - SchemaLoader.standardCFMD(KEYSPACE1, CF_STANDARDINT, IntegerType.instance)); + SchemaLoader.standardCFMD(KEYSPACE1, CF_STANDARD2)); + // TODO: Fix superCFMD failing on legacy table creation. Seems to be applying composite comparator to partition key + // SchemaLoader.superCFMD(KEYSPACE1, CF_SUPER1, LongType.instance)); + // SchemaLoader.superCFMD(KEYSPACE1, CF_SUPER6, "aaaaaaaa-bbbb-cccc-dddd-eeeeeeeeeeee", LexicalUUIDType.instance, UTF8Type.instance), SchemaLoader.createKeyspace(KEYSPACE2, SimpleStrategy.class, KSMetaData.optsWithRF(1), - SchemaLoader.standardCFMD(KEYSPACE2, CF_STANDARD1), - SchemaLoader.indexCFMD(KEYSPACE2, CF_INDEX1, true), - SchemaLoader.compositeIndexCFMD(KEYSPACE2, CF_INDEX2, true), - SchemaLoader.compositeIndexCFMD(KEYSPACE2, CF_INDEX3, true).gcGraceSeconds(0)); - SchemaLoader.createKeyspace(KEYSPACE3, - SimpleStrategy.class, - KSMetaData.optsWithRF(5), - SchemaLoader.indexCFMD(KEYSPACE3, CF_INDEX1, true)); - SchemaLoader.createKeyspace(KEYSPACE4, - SimpleStrategy.class, - KSMetaData.optsWithRF(1), - SchemaLoader.perRowIndexedCFMD(KEYSPACE4, "Indexed1")); + SchemaLoader.standardCFMD(KEYSPACE2, CF_STANDARD1)); + } + + @Before + public void truncateCFS() + { + Keyspace.open(KEYSPACE1).getColumnFamilyStore(CF_STANDARD1).truncateBlocking(); + Keyspace.open(KEYSPACE1).getColumnFamilyStore(CF_STANDARD2).truncateBlocking(); + // Keyspace.open(KEYSPACE1).getColumnFamilyStore(CF_SUPER1).truncateBlocking(); + + Keyspace.open(KEYSPACE2).getColumnFamilyStore(CF_STANDARD1).truncateBlocking(); } @Test @@ -175,21 +93,23 @@ public class ColumnFamilyStoreTest { Keyspace keyspace = Keyspace.open(KEYSPACE1); ColumnFamilyStore cfs = keyspace.getColumnFamilyStore(CF_STANDARD1); - cfs.truncateBlocking(); - Mutation rm; - rm = new Mutation(KEYSPACE1, ByteBufferUtil.bytes("key1")); - rm.add(CF_STANDARD1, cellname("Column1"), ByteBufferUtil.bytes("asdf"), 0); - rm.applyUnsafe(); + new RowUpdateBuilder(cfs.metadata, 0, "key1") + .clustering("Column1") + .add("val", "asdf") + .build() + .applyUnsafe(); cfs.forceBlockingFlush(); - rm = new Mutation(KEYSPACE1, ByteBufferUtil.bytes("key1")); - rm.add(CF_STANDARD1, cellname("Column1"), ByteBufferUtil.bytes("asdf"), 1); - rm.applyUnsafe(); + new RowUpdateBuilder(cfs.metadata, 1, "key1") + .clustering("Column1") + .add("val", "asdf") + .build() + .applyUnsafe(); cfs.forceBlockingFlush(); ((ClearableHistogram)cfs.metric.sstablesPerReadHistogram.cf).clear(); // resets counts - cfs.getColumnFamily(Util.namesQueryFilter(cfs, Util.dk("key1"), "Column1")); + Util.getAll(Util.cmd(cfs, "key1").includeRow("c1").build()); assertEquals(1, cfs.metric.sstablesPerReadHistogram.cf.getCount()); } @@ -198,788 +118,128 @@ public class ColumnFamilyStoreTest { Keyspace keyspace = Keyspace.open(KEYSPACE1); ColumnFamilyStore cfs = keyspace.getColumnFamilyStore(CF_STANDARD1); - cfs.truncateBlocking(); List rms = new LinkedList<>(); - Mutation rm = new Mutation(KEYSPACE1, ByteBufferUtil.bytes("key1")); - rm.add(CF_STANDARD1, cellname("Column1"), ByteBufferUtil.bytes("asdf"), 0); - rm.add(CF_STANDARD1, cellname("Column2"), ByteBufferUtil.bytes("asdf"), 0); - rms.add(rm); + rms.add(new RowUpdateBuilder(cfs.metadata, 0, "key1") + .clustering("Column1") + .add("val", "asdf") + .build()); + Util.writeColumnFamily(rms); List ssTables = keyspace.getAllSSTables(); assertEquals(1, ssTables.size()); ssTables.get(0).forceFilterFailures(); - ColumnFamily cf = cfs.getColumnFamily(QueryFilter.getIdentityFilter(Util.dk("key2"), CF_STANDARD1, System.currentTimeMillis())); - assertNull(cf); + Util.assertEmpty(Util.cmd(cfs, "key2").build()); } @Test public void testEmptyRow() throws Exception { Keyspace keyspace = Keyspace.open(KEYSPACE1); - final ColumnFamilyStore store = keyspace.getColumnFamilyStore(CF_STANDARD2); - Mutation rm; + final ColumnFamilyStore cfs = keyspace.getColumnFamilyStore(CF_STANDARD2); - rm = new Mutation(KEYSPACE1, ByteBufferUtil.bytes("key1")); - rm.delete(CF_STANDARD2, System.currentTimeMillis()); - rm.applyUnsafe(); + RowUpdateBuilder.deleteRow(cfs.metadata, FBUtilities.timestampMicros(), "key1", "Column1").applyUnsafe(); Runnable r = new WrappedRunnable() { public void runMayThrow() throws IOException { - QueryFilter sliceFilter = QueryFilter.getSliceFilter(Util.dk("key1"), CF_STANDARD2, Composites.EMPTY, Composites.EMPTY, false, 1, System.currentTimeMillis()); - ColumnFamily cf = store.getColumnFamily(sliceFilter); - assertTrue(cf.isMarkedForDelete()); - assertFalse(cf.hasColumns()); - - QueryFilter namesFilter = Util.namesQueryFilter(store, Util.dk("key1"), "a"); - cf = store.getColumnFamily(namesFilter); - assertTrue(cf.isMarkedForDelete()); - assertFalse(cf.hasColumns()); + Row toCheck = Util.getOnlyRowUnfiltered(Util.cmd(cfs, "key1").build()); + Iterator iter = toCheck.iterator(); + assert(Iterators.size(iter) == 0); } }; - KeyspaceTest.reTest(store, r); + reTest(cfs, r); } - @Test - public void testSkipStartKey() - { - ColumnFamilyStore cfs = insertKey1Key2(); - - IPartitioner p = StorageService.getPartitioner(); - List result = cfs.getRangeSlice(Util.range(p, "key1", "key2"), - null, - Util.namesFilter(cfs, "asdf"), - 10); - assertEquals(1, result.size()); - assert result.get(0).key.getKey().equals(ByteBufferUtil.bytes("key2")); - } - - @Test - public void testIndexScan() - { - ColumnFamilyStore cfs = Keyspace.open(KEYSPACE1).getColumnFamilyStore(CF_INDEX1); - Mutation rm; - CellName nobirthdate = cellname("notbirthdate"); - CellName birthdate = cellname("birthdate"); - - rm = new Mutation(KEYSPACE1, ByteBufferUtil.bytes("k1")); - rm.add(CF_INDEX1, nobirthdate, ByteBufferUtil.bytes(1L), 0); - rm.add(CF_INDEX1, birthdate, ByteBufferUtil.bytes(1L), 0); - rm.applyUnsafe(); - - rm = new Mutation(KEYSPACE1, ByteBufferUtil.bytes("k2")); - rm.add(CF_INDEX1, nobirthdate, ByteBufferUtil.bytes(2L), 0); - rm.add(CF_INDEX1, birthdate, ByteBufferUtil.bytes(2L), 0); - rm.applyUnsafe(); - - rm = new Mutation(KEYSPACE1, ByteBufferUtil.bytes("k3")); - rm.add(CF_INDEX1, nobirthdate, ByteBufferUtil.bytes(2L), 0); - rm.add(CF_INDEX1, birthdate, ByteBufferUtil.bytes(1L), 0); - rm.applyUnsafe(); - - rm = new Mutation(KEYSPACE1, ByteBufferUtil.bytes("k4aaaa")); - rm.add(CF_INDEX1, nobirthdate, ByteBufferUtil.bytes(2L), 0); - rm.add(CF_INDEX1, birthdate, ByteBufferUtil.bytes(3L), 0); - rm.applyUnsafe(); - - // basic single-expression query - IndexExpression expr = new IndexExpression(ByteBufferUtil.bytes("birthdate"), Operator.EQ, ByteBufferUtil.bytes(1L)); - List clause = Arrays.asList(expr); - IDiskAtomFilter filter = new IdentityQueryFilter(); - Range range = Util.range("", ""); - List rows = cfs.search(range, clause, filter, 100); - - assert rows != null; - assert rows.size() == 2 : StringUtils.join(rows, ","); - - String key = new String(rows.get(0).key.getKey().array(), rows.get(0).key.getKey().position(), rows.get(0).key.getKey().remaining()); - assert "k1".equals( key ) : key; - - key = new String(rows.get(1).key.getKey().array(), rows.get(1).key.getKey().position(), rows.get(1).key.getKey().remaining()); - assert "k3".equals(key) : key; - - assert ByteBufferUtil.bytes(1L).equals( rows.get(0).cf.getColumn(birthdate).value()); - assert ByteBufferUtil.bytes(1L).equals( rows.get(1).cf.getColumn(birthdate).value()); - - // add a second expression - IndexExpression expr2 = new IndexExpression(ByteBufferUtil.bytes("notbirthdate"), Operator.GTE, ByteBufferUtil.bytes(2L)); - clause = Arrays.asList(expr, expr2); - rows = cfs.search(range, clause, filter, 100); - - assert rows.size() == 1 : StringUtils.join(rows, ","); - key = new String(rows.get(0).key.getKey().array(), rows.get(0).key.getKey().position(), rows.get(0).key.getKey().remaining()); - assert "k3".equals( key ); - - // same query again, but with resultset not including the subordinate expression - rows = cfs.search(range, clause, Util.namesFilter(cfs, "birthdate"), 100); - - assert rows.size() == 1 : StringUtils.join(rows, ","); - key = new String(rows.get(0).key.getKey().array(), rows.get(0).key.getKey().position(), rows.get(0).key.getKey().remaining()); - assert "k3".equals( key ); - - assert rows.get(0).cf.getColumnCount() == 1 : rows.get(0).cf; - - // once more, this time with a slice rowset that needs to be expanded - SliceQueryFilter emptyFilter = new SliceQueryFilter(Composites.EMPTY, Composites.EMPTY, false, 0); - rows = cfs.search(range, clause, emptyFilter, 100); - - assert rows.size() == 1 : StringUtils.join(rows, ","); - key = new String(rows.get(0).key.getKey().array(), rows.get(0).key.getKey().position(), rows.get(0).key.getKey().remaining()); - assert "k3".equals( key ); - - assertFalse(rows.get(0).cf.hasColumns()); - - // query with index hit but rejected by secondary clause, with a small enough count that just checking count - // doesn't tell the scan loop that it's done - IndexExpression expr3 = new IndexExpression(ByteBufferUtil.bytes("notbirthdate"), Operator.EQ, ByteBufferUtil.bytes(-1L)); - clause = Arrays.asList(expr, expr3); - rows = cfs.search(range, clause, filter, 100); - - assert rows.isEmpty(); - } - - @Test - public void testLargeScan() - { - Mutation rm; - ColumnFamilyStore cfs = Keyspace.open(KEYSPACE1).getColumnFamilyStore(CF_INDEX1); - for (int i = 0; i < 100; i++) - { - rm = new Mutation(KEYSPACE1, ByteBufferUtil.bytes("key" + i)); - rm.add(CF_INDEX1, cellname("birthdate"), ByteBufferUtil.bytes(34L), 0); - rm.add(CF_INDEX1, cellname("notbirthdate"), ByteBufferUtil.bytes((long) (i % 2)), 0); - rm.applyUnsafe(); - } - - IndexExpression expr = new IndexExpression(ByteBufferUtil.bytes("birthdate"), Operator.EQ, ByteBufferUtil.bytes(34L)); - IndexExpression expr2 = new IndexExpression(ByteBufferUtil.bytes("notbirthdate"), Operator.EQ, ByteBufferUtil.bytes(1L)); - List clause = Arrays.asList(expr, expr2); - IDiskAtomFilter filter = new IdentityQueryFilter(); - Range range = Util.range("", ""); - List rows = cfs.search(range, clause, filter, 100); - - assert rows != null; - assert rows.size() == 50 : rows.size(); - Set keys = new HashSet(); - // extra check that there are no duplicate results -- see https://issues.apache.org/jira/browse/CASSANDRA-2406 - for (Row row : rows) - keys.add(row.key); - assert rows.size() == keys.size(); - } - - @Test - public void testIndexDeletions() throws IOException - { - ColumnFamilyStore cfs = Keyspace.open(KEYSPACE3).getColumnFamilyStore(CF_INDEX1); - Mutation rm; - - rm = new Mutation(KEYSPACE3, ByteBufferUtil.bytes("k1")); - rm.add(CF_INDEX1, cellname("birthdate"), ByteBufferUtil.bytes(1L), 0); - rm.applyUnsafe(); - - IndexExpression expr = new IndexExpression(ByteBufferUtil.bytes("birthdate"), Operator.EQ, ByteBufferUtil.bytes(1L)); - List clause = Arrays.asList(expr); - IDiskAtomFilter filter = new IdentityQueryFilter(); - Range range = Util.range("", ""); - List rows = cfs.search(range, clause, filter, 100); - assert rows.size() == 1 : StringUtils.join(rows, ","); - String key = ByteBufferUtil.string(rows.get(0).key.getKey()); - assert "k1".equals( key ); - - // delete the column directly - rm = new Mutation(KEYSPACE3, ByteBufferUtil.bytes("k1")); - rm.delete(CF_INDEX1, cellname("birthdate"), 1); - rm.applyUnsafe(); - rows = cfs.search(range, clause, filter, 100); - assert rows.isEmpty(); - - // verify that it's not being indexed under the deletion column value either - Cell deletion = rm.getColumnFamilies().iterator().next().iterator().next(); - ByteBuffer deletionLong = ByteBufferUtil.bytes((long) ByteBufferUtil.toInt(deletion.value())); - IndexExpression expr0 = new IndexExpression(ByteBufferUtil.bytes("birthdate"), Operator.EQ, deletionLong); - List clause0 = Arrays.asList(expr0); - rows = cfs.search(range, clause0, filter, 100); - assert rows.isEmpty(); - - // resurrect w/ a newer timestamp - rm = new Mutation(KEYSPACE3, ByteBufferUtil.bytes("k1")); - rm.add(CF_INDEX1, cellname("birthdate"), ByteBufferUtil.bytes(1L), 2); - rm.applyUnsafe(); - rows = cfs.search(range, clause, filter, 100); - assert rows.size() == 1 : StringUtils.join(rows, ","); - key = ByteBufferUtil.string(rows.get(0).key.getKey()); - assert "k1".equals( key ); - - // verify that row and delete w/ older timestamp does nothing - rm = new Mutation(KEYSPACE3, ByteBufferUtil.bytes("k1")); - rm.delete(CF_INDEX1, 1); - rm.applyUnsafe(); - rows = cfs.search(range, clause, filter, 100); - assert rows.size() == 1 : StringUtils.join(rows, ","); - key = ByteBufferUtil.string(rows.get(0).key.getKey()); - assert "k1".equals( key ); - - // similarly, column delete w/ older timestamp should do nothing - rm = new Mutation(KEYSPACE3, ByteBufferUtil.bytes("k1")); - rm.delete(CF_INDEX1, cellname("birthdate"), 1); - rm.applyUnsafe(); - rows = cfs.search(range, clause, filter, 100); - assert rows.size() == 1 : StringUtils.join(rows, ","); - key = ByteBufferUtil.string(rows.get(0).key.getKey()); - assert "k1".equals( key ); - - // delete the entire row (w/ newer timestamp this time) - rm = new Mutation(KEYSPACE3, ByteBufferUtil.bytes("k1")); - rm.delete(CF_INDEX1, 3); - rm.applyUnsafe(); - rows = cfs.search(range, clause, filter, 100); - assert rows.isEmpty() : StringUtils.join(rows, ","); - - // make sure obsolete mutations don't generate an index entry - rm = new Mutation(KEYSPACE3, ByteBufferUtil.bytes("k1")); - rm.add(CF_INDEX1, cellname("birthdate"), ByteBufferUtil.bytes(1L), 3); - rm.applyUnsafe(); - rows = cfs.search(range, clause, filter, 100); - assert rows.isEmpty() : StringUtils.join(rows, ","); - - // try insert followed by row delete in the same mutation - rm = new Mutation(KEYSPACE3, ByteBufferUtil.bytes("k1")); - rm.add(CF_INDEX1, cellname("birthdate"), ByteBufferUtil.bytes(1L), 1); - rm.delete(CF_INDEX1, 2); - rm.applyUnsafe(); - rows = cfs.search(range, clause, filter, 100); - assert rows.isEmpty() : StringUtils.join(rows, ","); - - // try row delete followed by insert in the same mutation - rm = new Mutation(KEYSPACE3, ByteBufferUtil.bytes("k1")); - rm.delete(CF_INDEX1, 3); - rm.add(CF_INDEX1, cellname("birthdate"), ByteBufferUtil.bytes(1L), 4); - rm.applyUnsafe(); - rows = cfs.search(range, clause, filter, 100); - assert rows.size() == 1 : StringUtils.join(rows, ","); - key = ByteBufferUtil.string(rows.get(0).key.getKey()); - assert "k1".equals( key ); - } - - @Test - public void testIndexUpdate() throws IOException - { - Keyspace keyspace = Keyspace.open(KEYSPACE2); - ColumnFamilyStore cfs = keyspace.getColumnFamilyStore(CF_INDEX1); - CellName birthdate = cellname("birthdate"); - - // create a row and update the birthdate value, test that the index query fetches the new version - Mutation rm; - rm = new Mutation(KEYSPACE2, ByteBufferUtil.bytes("k1")); - rm.add(CF_INDEX1, birthdate, ByteBufferUtil.bytes(1L), 1); - rm.applyUnsafe(); - rm = new Mutation(KEYSPACE2, ByteBufferUtil.bytes("k1")); - rm.add(CF_INDEX1, birthdate, ByteBufferUtil.bytes(2L), 2); - rm.applyUnsafe(); - - IndexExpression expr = new IndexExpression(ByteBufferUtil.bytes("birthdate"), Operator.EQ, ByteBufferUtil.bytes(1L)); - List clause = Arrays.asList(expr); - IDiskAtomFilter filter = new IdentityQueryFilter(); - Range range = Util.range("", ""); - List rows = cfs.search(range, clause, filter, 100); - assert rows.size() == 0; - - expr = new IndexExpression(ByteBufferUtil.bytes("birthdate"), Operator.EQ, ByteBufferUtil.bytes(2L)); - clause = Arrays.asList(expr); - rows = keyspace.getColumnFamilyStore(CF_INDEX1).search(range, clause, filter, 100); - String key = ByteBufferUtil.string(rows.get(0).key.getKey()); - assert "k1".equals( key ); - - // update the birthdate value with an OLDER timestamp, and test that the index ignores this - rm = new Mutation(KEYSPACE2, ByteBufferUtil.bytes("k1")); - rm.add(CF_INDEX1, birthdate, ByteBufferUtil.bytes(3L), 0); - rm.applyUnsafe(); - - rows = keyspace.getColumnFamilyStore(CF_INDEX1).search(range, clause, filter, 100); - key = ByteBufferUtil.string(rows.get(0).key.getKey()); - assert "k1".equals( key ); - - } - - @Test - public void testIndexUpdateOverwritingExpiringColumns() throws Exception - { - // see CASSANDRA-7268 - Keyspace keyspace = Keyspace.open(KEYSPACE2); - - // create a row and update the birthdate value with an expiring column - Mutation rm; - rm = new Mutation(KEYSPACE2, ByteBufferUtil.bytes("k100")); - rm.add("Indexed1", cellname("birthdate"), ByteBufferUtil.bytes(100L), 1, 1000); - rm.applyUnsafe(); - - IndexExpression expr = new IndexExpression(ByteBufferUtil.bytes("birthdate"), Operator.EQ, ByteBufferUtil.bytes(100L)); - List clause = Arrays.asList(expr); - IDiskAtomFilter filter = new IdentityQueryFilter(); - Range range = Util.range("", ""); - List rows = keyspace.getColumnFamilyStore("Indexed1").search(range, clause, filter, 100); - assertEquals(1, rows.size()); - - // requires a 1s sleep because we calculate local expiry time as (now() / 1000) + ttl - TimeUnit.SECONDS.sleep(1); - - // now overwrite with the same name/value/ttl, but the local expiry time will be different - rm = new Mutation(KEYSPACE2, ByteBufferUtil.bytes("k100")); - rm.add("Indexed1", cellname("birthdate"), ByteBufferUtil.bytes(100L), 1, 1000); - rm.applyUnsafe(); - - rows = keyspace.getColumnFamilyStore("Indexed1").search(range, clause, filter, 100); - assertEquals(1, rows.size()); - - // check that modifying the indexed value using the same timestamp behaves as expected - rm = new Mutation(KEYSPACE2, ByteBufferUtil.bytes("k101")); - rm.add("Indexed1", cellname("birthdate"), ByteBufferUtil.bytes(101L), 1, 1000); - rm.applyUnsafe(); - - expr = new IndexExpression(ByteBufferUtil.bytes("birthdate"), Operator.EQ, ByteBufferUtil.bytes(101L)); - clause = Arrays.asList(expr); - rows = keyspace.getColumnFamilyStore("Indexed1").search(range, clause, filter, 100); - assertEquals(1, rows.size()); - - TimeUnit.SECONDS.sleep(1); - rm = new Mutation(KEYSPACE2, ByteBufferUtil.bytes("k101")); - rm.add("Indexed1", cellname("birthdate"), ByteBufferUtil.bytes(102L), 1, 1000); - rm.applyUnsafe(); - // search for the old value - rows = keyspace.getColumnFamilyStore("Indexed1").search(range, clause, filter, 100); - assertEquals(0, rows.size()); - // and for the new - expr = new IndexExpression(ByteBufferUtil.bytes("birthdate"), Operator.EQ, ByteBufferUtil.bytes(102L)); - clause = Arrays.asList(expr); - rows = keyspace.getColumnFamilyStore("Indexed1").search(range, clause, filter, 100); - assertEquals(1, rows.size()); - } - - @Test - public void testDeleteOfInconsistentValuesInKeysIndex() throws Exception - { - String keySpace = KEYSPACE2; - String cfName = CF_INDEX1; - - Keyspace keyspace = Keyspace.open(keySpace); - ColumnFamilyStore cfs = keyspace.getColumnFamilyStore(cfName); - cfs.truncateBlocking(); - - ByteBuffer rowKey = ByteBufferUtil.bytes("k1"); - CellName colName = cellname("birthdate"); - ByteBuffer val1 = ByteBufferUtil.bytes(1L); - ByteBuffer val2 = ByteBufferUtil.bytes(2L); - - // create a row and update the "birthdate" value, test that the index query fetches this version - Mutation rm; - rm = new Mutation(keySpace, rowKey); - rm.add(cfName, colName, val1, 0); - rm.applyUnsafe(); - IndexExpression expr = new IndexExpression(ByteBufferUtil.bytes("birthdate"), Operator.EQ, val1); - List clause = Arrays.asList(expr); - IDiskAtomFilter filter = new IdentityQueryFilter(); - Range range = Util.range("", ""); - List rows = keyspace.getColumnFamilyStore(cfName).search(range, clause, filter, 100); - assertEquals(1, rows.size()); - - // force a flush, so our index isn't being read from a memtable - keyspace.getColumnFamilyStore(cfName).forceBlockingFlush(); - - // now apply another update, but force the index update to be skipped - rm = new Mutation(keySpace, rowKey); - rm.add(cfName, colName, val2, 1); - keyspace.apply(rm, true, false); - - // Now searching the index for either the old or new value should return 0 rows - // because the new value was not indexed and the old value should be ignored - // (and in fact purged from the index cf). - // first check for the old value - rows = keyspace.getColumnFamilyStore(cfName).search(range, clause, filter, 100); - assertEquals(0, rows.size()); - // now check for the updated value - expr = new IndexExpression(ByteBufferUtil.bytes("birthdate"), Operator.EQ, val2); - clause = Arrays.asList(expr); - filter = new IdentityQueryFilter(); - range = Util.range("", ""); - rows = keyspace.getColumnFamilyStore(cfName).search(range, clause, filter, 100); - assertEquals(0, rows.size()); - - // now, reset back to the original value, still skipping the index update, to - // make sure the value was expunged from the index when it was discovered to be inconsistent - rm = new Mutation(keySpace, rowKey); - rm.add(cfName, colName, ByteBufferUtil.bytes(1L), 3); - keyspace.apply(rm, true, false); - - expr = new IndexExpression(ByteBufferUtil.bytes("birthdate"), Operator.EQ, ByteBufferUtil.bytes(1L)); - clause = Arrays.asList(expr); - filter = new IdentityQueryFilter(); - range = Util.range("", ""); - rows = keyspace.getColumnFamilyStore(cfName).search(range, clause, filter, 100); - assertEquals(0, rows.size()); - } - - @Test - public void testDeleteOfInconsistentValuesFromCompositeIndex() throws Exception - { - String keySpace = KEYSPACE2; - String cfName = CF_INDEX2; - - Keyspace keyspace = Keyspace.open(keySpace); - ColumnFamilyStore cfs = keyspace.getColumnFamilyStore(cfName); - cfs.truncateBlocking(); - - ByteBuffer rowKey = ByteBufferUtil.bytes("k1"); - ByteBuffer clusterKey = ByteBufferUtil.bytes("ck1"); - ByteBuffer colName = ByteBufferUtil.bytes("col1"); - - CellNameType baseComparator = cfs.getComparator(); - CellName compositeName = baseComparator.makeCellName(clusterKey, colName); - - ByteBuffer val1 = ByteBufferUtil.bytes("v1"); - ByteBuffer val2 = ByteBufferUtil.bytes("v2"); - - // create a row and update the author value - Mutation rm; - rm = new Mutation(keySpace, rowKey); - rm.add(cfName, compositeName, val1, 0); - rm.applyUnsafe(); - - // test that the index query fetches this version - IndexExpression expr = new IndexExpression(colName, Operator.EQ, val1); - List clause = Arrays.asList(expr); - IDiskAtomFilter filter = new IdentityQueryFilter(); - Range range = Util.range("", ""); - List rows = keyspace.getColumnFamilyStore(cfName).search(range, clause, filter, 100); - assertEquals(1, rows.size()); - - // force a flush and retry the query, so our index isn't being read from a memtable - keyspace.getColumnFamilyStore(cfName).forceBlockingFlush(); - rows = keyspace.getColumnFamilyStore(cfName).search(range, clause, filter, 100); - assertEquals(1, rows.size()); - - // now apply another update, but force the index update to be skipped - rm = new Mutation(keySpace, rowKey); - rm.add(cfName, compositeName, val2, 1); - keyspace.apply(rm, true, false); - - // Now searching the index for either the old or new value should return 0 rows - // because the new value was not indexed and the old value should be ignored - // (and in fact purged from the index cf). - // first check for the old value - rows = keyspace.getColumnFamilyStore(cfName).search(range, clause, filter, 100); - assertEquals(0, rows.size()); - // now check for the updated value - expr = new IndexExpression(colName, Operator.EQ, val2); - clause = Arrays.asList(expr); - filter = new IdentityQueryFilter(); - range = Util.range("", ""); - rows = keyspace.getColumnFamilyStore(cfName).search(range, clause, filter, 100); - assertEquals(0, rows.size()); - - // now, reset back to the original value, still skipping the index update, to - // make sure the value was expunged from the index when it was discovered to be inconsistent - rm = new Mutation(keySpace, rowKey); - rm.add(cfName, compositeName, val1, 2); - keyspace.apply(rm, true, false); - - expr = new IndexExpression(colName, Operator.EQ, val1); - clause = Arrays.asList(expr); - filter = new IdentityQueryFilter(); - range = Util.range("", ""); - rows = keyspace.getColumnFamilyStore(cfName).search(range, clause, filter, 100); - assertEquals(0, rows.size()); - } - - // See CASSANDRA-6098 - @Test - public void testDeleteCompositeIndex() throws Exception - { - String keySpace = KEYSPACE2; - String cfName = CF_INDEX3; // has gcGrace 0 - - Keyspace keyspace = Keyspace.open(keySpace); - ColumnFamilyStore cfs = keyspace.getColumnFamilyStore(cfName); - cfs.truncateBlocking(); - - ByteBuffer rowKey = ByteBufferUtil.bytes("k1"); - ByteBuffer clusterKey = ByteBufferUtil.bytes("ck1"); - ByteBuffer colName = ByteBufferUtil.bytes("col1"); - - CellNameType baseComparator = cfs.getComparator(); - CellName compositeName = baseComparator.makeCellName(clusterKey, colName); - - ByteBuffer val1 = ByteBufferUtil.bytes("v2"); - - // Insert indexed value. - Mutation rm; - rm = new Mutation(keySpace, rowKey); - rm.add(cfName, compositeName, val1, 0); - rm.applyUnsafe(); - - // Now delete the value and flush too. - rm = new Mutation(keySpace, rowKey); - rm.delete(cfName, 1); - rm.applyUnsafe(); - - // We want the data to be gcable, but even if gcGrace == 0, we still need to wait 1 second - // since we won't gc on a tie. - try { Thread.sleep(1000); } catch (Exception e) {} - - // Read the index and we check we do get no value (and no NPE) - // Note: the index will return the entry because it hasn't been deleted (we - // haven't read yet nor compacted) but the data read itself will return null - IndexExpression expr = new IndexExpression(colName, Operator.EQ, val1); - List clause = Arrays.asList(expr); - IDiskAtomFilter filter = new IdentityQueryFilter(); - Range range = Util.range("", ""); - List rows = keyspace.getColumnFamilyStore(cfName).search(range, clause, filter, 100); - assertEquals(0, rows.size()); - } - - // See CASSANDRA-2628 - @Test - public void testIndexScanWithLimitOne() - { - ColumnFamilyStore cfs = Keyspace.open(KEYSPACE1).getColumnFamilyStore(CF_INDEX1); - Mutation rm; - - CellName nobirthdate = cellname("notbirthdate"); - CellName birthdate = cellname("birthdate"); - - rm = new Mutation(KEYSPACE1, ByteBufferUtil.bytes("kk1")); - rm.add(CF_INDEX1, nobirthdate, ByteBufferUtil.bytes(1L), 0); - rm.add(CF_INDEX1, birthdate, ByteBufferUtil.bytes(1L), 0); - rm.applyUnsafe(); - - rm = new Mutation(KEYSPACE1, ByteBufferUtil.bytes("kk2")); - rm.add(CF_INDEX1, nobirthdate, ByteBufferUtil.bytes(2L), 0); - rm.add(CF_INDEX1, birthdate, ByteBufferUtil.bytes(1L), 0); - rm.applyUnsafe(); - - rm = new Mutation(KEYSPACE1, ByteBufferUtil.bytes("kk3")); - rm.add(CF_INDEX1, nobirthdate, ByteBufferUtil.bytes(2L), 0); - rm.add(CF_INDEX1, birthdate, ByteBufferUtil.bytes(1L), 0); - rm.applyUnsafe(); - - rm = new Mutation(KEYSPACE1, ByteBufferUtil.bytes("kk4")); - rm.add(CF_INDEX1, nobirthdate, ByteBufferUtil.bytes(2L), 0); - rm.add(CF_INDEX1, birthdate, ByteBufferUtil.bytes(1L), 0); - rm.applyUnsafe(); - - // basic single-expression query - IndexExpression expr1 = new IndexExpression(ByteBufferUtil.bytes("birthdate"), Operator.EQ, ByteBufferUtil.bytes(1L)); - IndexExpression expr2 = new IndexExpression(ByteBufferUtil.bytes("notbirthdate"), Operator.GT, ByteBufferUtil.bytes(1L)); - List clause = Arrays.asList(expr1, expr2); - IDiskAtomFilter filter = new IdentityQueryFilter(); - Range range = Util.range("", ""); - List rows = cfs.search(range, clause, filter, 1); - - assert rows != null; - assert rows.size() == 1 : StringUtils.join(rows, ","); - } - - @Test - public void testIndexCreate() throws IOException, InterruptedException, ExecutionException - { - Keyspace keyspace = Keyspace.open(KEYSPACE1); - ColumnFamilyStore cfs = keyspace.getColumnFamilyStore(CF_INDEX2); - - // create a row and update the birthdate value, test that the index query fetches the new version - Mutation rm; - rm = new Mutation(KEYSPACE1, ByteBufferUtil.bytes("k1")); - rm.add(CF_INDEX2, cellname("birthdate"), ByteBufferUtil.bytes(1L), 1); - rm.applyUnsafe(); - - ColumnDefinition old = cfs.metadata.getColumnDefinition(ByteBufferUtil.bytes("birthdate")); - ColumnDefinition cd = ColumnDefinition.regularDef(cfs.metadata, old.name.bytes, old.type, null).setIndex("birthdate_index", IndexType.KEYS, null); - Future future = cfs.indexManager.addIndexedColumn(cd); - future.get(); - // we had a bug (CASSANDRA-2244) where index would get created but not flushed -- check for that - assert cfs.indexManager.getIndexForColumn(cd.name.bytes).getIndexCfs().getSSTables().size() > 0; - - queryBirthdate(keyspace); - - // validate that drop clears it out & rebuild works (CASSANDRA-2320) - SecondaryIndex indexedCfs = cfs.indexManager.getIndexForColumn(ByteBufferUtil.bytes("birthdate")); - cfs.indexManager.removeIndexedColumn(ByteBufferUtil.bytes("birthdate")); - assert !indexedCfs.isIndexBuilt(ByteBufferUtil.bytes("birthdate")); - - // rebuild & re-query - future = cfs.indexManager.addIndexedColumn(cd); - future.get(); - queryBirthdate(keyspace); - } - - private void queryBirthdate(Keyspace keyspace) throws CharacterCodingException - { - IndexExpression expr = new IndexExpression(ByteBufferUtil.bytes("birthdate"), Operator.EQ, ByteBufferUtil.bytes(1L)); - List clause = Arrays.asList(expr); - IDiskAtomFilter filter = new IdentityQueryFilter(); - List rows = keyspace.getColumnFamilyStore(CF_INDEX2).search(Util.range("", ""), clause, filter, 100); - assert rows.size() == 1 : StringUtils.join(rows, ","); - assertEquals("k1", ByteBufferUtil.string(rows.get(0).key.getKey())); - } - - @Test - public void testCassandra6778() throws CharacterCodingException - { - String cfname = CF_STANDARDINT; - Keyspace keyspace = Keyspace.open(KEYSPACE1); - ColumnFamilyStore cfs = keyspace.getColumnFamilyStore(cfname); - - // insert two columns that represent the same integer but have different binary forms (the - // second one is padded with extra zeros) - Mutation rm = new Mutation(KEYSPACE1, ByteBufferUtil.bytes("k1")); - CellName column1 = cellname(ByteBuffer.wrap(new byte[]{1})); - rm.add(cfname, column1, ByteBufferUtil.bytes("data1"), 1); - rm.applyUnsafe(); - cfs.forceBlockingFlush(); - - rm = new Mutation(KEYSPACE1, ByteBufferUtil.bytes("k1")); - CellName column2 = cellname(ByteBuffer.wrap(new byte[]{0, 0, 1})); - rm.add(cfname, column2, ByteBufferUtil.bytes("data2"), 2); - rm.applyUnsafe(); - cfs.forceBlockingFlush(); - - // fetch by the first column name; we should get the second version of the column value - SliceByNamesReadCommand cmd = new SliceByNamesReadCommand( - KEYSPACE1, ByteBufferUtil.bytes("k1"), cfname, System.currentTimeMillis(), - new NamesQueryFilter(FBUtilities.singleton(column1, cfs.getComparator()))); - - ColumnFamily cf = cmd.getRow(keyspace).cf; - assertEquals(1, cf.getColumnCount()); - Cell cell = cf.getColumn(column1); - assertEquals("data2", ByteBufferUtil.string(cell.value())); - assertEquals(column2, cell.name()); - - // fetch by the second column name; we should get the second version of the column value - cmd = new SliceByNamesReadCommand( - KEYSPACE1, ByteBufferUtil.bytes("k1"), cfname, System.currentTimeMillis(), - new NamesQueryFilter(FBUtilities.singleton(column2, cfs.getComparator()))); - - cf = cmd.getRow(keyspace).cf; - assertEquals(1, cf.getColumnCount()); - cell = cf.getColumn(column2); - assertEquals("data2", ByteBufferUtil.string(cell.value())); - assertEquals(column2, cell.name()); - } - - @Test - public void testInclusiveBounds() - { - ColumnFamilyStore cfs = insertKey1Key2(); - - List result = cfs.getRangeSlice(Util.bounds("key1", "key2"), - null, - Util.namesFilter(cfs, "asdf"), - 10); - assertEquals(2, result.size()); - assert result.get(0).key.getKey().equals(ByteBufferUtil.bytes("key1")); - } - - @Test - public void testDeleteSuperRowSticksAfterFlush() throws Throwable - { - String keyspaceName = KEYSPACE1; - String cfName= CF_SUPER1; - ByteBuffer scfName = ByteBufferUtil.bytes("SuperDuper"); - Keyspace keyspace = Keyspace.open(keyspaceName); - ColumnFamilyStore cfs = keyspace.getColumnFamilyStore(cfName); - DecoratedKey key = Util.dk("flush-resurrection"); - - // create an isolated sstable. - putColsSuper(cfs, key, scfName, - new BufferCell(cellname(1L), ByteBufferUtil.bytes("val1"), 1), - new BufferCell(cellname(2L), ByteBufferUtil.bytes("val2"), 1), - new BufferCell(cellname(3L), ByteBufferUtil.bytes("val3"), 1)); - cfs.forceBlockingFlush(); - - // insert, don't flush. - putColsSuper(cfs, key, scfName, - new BufferCell(cellname(4L), ByteBufferUtil.bytes("val4"), 1), - new BufferCell(cellname(5L), ByteBufferUtil.bytes("val5"), 1), - new BufferCell(cellname(6L), ByteBufferUtil.bytes("val6"), 1)); - - // verify insert. - final SlicePredicate sp = new SlicePredicate(); - sp.setSlice_range(new SliceRange()); - sp.getSlice_range().setCount(100); - sp.getSlice_range().setStart(ArrayUtils.EMPTY_BYTE_ARRAY); - sp.getSlice_range().setFinish(ArrayUtils.EMPTY_BYTE_ARRAY); - - assertRowAndColCount(1, 6, false, cfs.getRangeSlice(Util.range("f", "g"), null, ThriftValidation.asIFilter(sp, cfs.metadata, scfName), 100)); - - // delete - Mutation rm = new Mutation(keyspace.getName(), key.getKey()); - rm.deleteRange(cfName, SuperColumns.startOf(scfName), SuperColumns.endOf(scfName), 2); - rm.applyUnsafe(); - - // verify delete. - assertRowAndColCount(1, 0, false, cfs.getRangeSlice(Util.range("f", "g"), null, ThriftValidation.asIFilter(sp, cfs.metadata, scfName), 100)); - - // flush - cfs.forceBlockingFlush(); - - // re-verify delete. - assertRowAndColCount(1, 0, false, cfs.getRangeSlice(Util.range("f", "g"), null, ThriftValidation.asIFilter(sp, cfs.metadata, scfName), 100)); - - // late insert. - putColsSuper(cfs, key, scfName, - new BufferCell(cellname(4L), ByteBufferUtil.bytes("val4"), 1L), - new BufferCell(cellname(7L), ByteBufferUtil.bytes("val7"), 1L)); - - // re-verify delete. - assertRowAndColCount(1, 0, false, cfs.getRangeSlice(Util.range("f", "g"), null, ThriftValidation.asIFilter(sp, cfs.metadata, scfName), 100)); - - // make sure new writes are recognized. - putColsSuper(cfs, key, scfName, - new BufferCell(cellname(3L), ByteBufferUtil.bytes("val3"), 3), - new BufferCell(cellname(8L), ByteBufferUtil.bytes("val8"), 3), - new BufferCell(cellname(9L), ByteBufferUtil.bytes("val9"), 3)); - assertRowAndColCount(1, 3, false, cfs.getRangeSlice(Util.range("f", "g"), null, ThriftValidation.asIFilter(sp, cfs.metadata, scfName), 100)); - } - - private static void assertRowAndColCount(int rowCount, int colCount, boolean isDeleted, Collection rows) throws CharacterCodingException - { - assert rows.size() == rowCount : "rowcount " + rows.size(); - for (Row row : rows) - { - assert row.cf != null : "cf was null"; - assert row.cf.getColumnCount() == colCount : "colcount " + row.cf.getColumnCount() + "|" + str(row.cf); - if (isDeleted) - assert row.cf.isMarkedForDelete() : "cf not marked for delete"; - } - } - - private static String str(ColumnFamily cf) throws CharacterCodingException - { - StringBuilder sb = new StringBuilder(); - for (Cell col : cf.getSortedColumns()) - sb.append(String.format("(%s,%s,%d),", ByteBufferUtil.string(col.name().toByteBuffer()), ByteBufferUtil.string(col.value()), col.timestamp())); - return sb.toString(); - } - - private static void putColsSuper(ColumnFamilyStore cfs, DecoratedKey key, ByteBuffer scfName, Cell... cols) throws Throwable - { - ColumnFamily cf = ArrayBackedSortedColumns.factory.create(cfs.keyspace.getName(), cfs.name); - for (Cell col : cols) - cf.addColumn(col.withUpdatedName(CellNames.compositeDense(scfName, col.name().toByteBuffer()))); - Mutation rm = new Mutation(cfs.keyspace.getName(), key.getKey(), cf); - rm.applyUnsafe(); - } - - private static void putColsStandard(ColumnFamilyStore cfs, DecoratedKey key, Cell... cols) throws Throwable - { - ColumnFamily cf = ArrayBackedSortedColumns.factory.create(cfs.keyspace.getName(), cfs.name); - for (Cell col : cols) - cf.addColumn(col); - Mutation rm = new Mutation(cfs.keyspace.getName(), key.getKey(), cf); - rm.applyUnsafe(); - } + // TODO: Implement this once we have hooks to super columns available in CQL context +// @Test +// public void testDeleteSuperRowSticksAfterFlush() throws Throwable +// { +// String keyspaceName = KEYSPACE1; +// String cfName= CF_SUPER1; +// +// Keyspace keyspace = Keyspace.open(keyspaceName); +// ColumnFamilyStore cfs = keyspace.getColumnFamilyStore(cfName); +// +// ByteBuffer scfName = ByteBufferUtil.bytes("SuperDuper"); +// DecoratedKey key = Util.dk("flush-resurrection"); +// +// // create an isolated sstable. +// putColSuper(cfs, key, 0, ByteBufferUtil.bytes("val"), ByteBufferUtil.bytes(1L), ByteBufferUtil.bytes(1L), ByteBufferUtil.bytes("val1")); + +// putColsSuper(cfs, key, scfName, +// new BufferCell(cellname(1L), ByteBufferUtil.bytes("val1"), 1), +// new BufferCell(cellname(2L), ByteBufferUtil.bytes("val2"), 1), +// new BufferCell(cellname(3L), ByteBufferUtil.bytes("val3"), 1)); +// cfs.forceBlockingFlush(); +// +// // insert, don't flush. +// putColsSuper(cfs, key, scfName, +// new BufferCell(cellname(4L), ByteBufferUtil.bytes("val4"), 1), +// new BufferCell(cellname(5L), ByteBufferUtil.bytes("val5"), 1), +// new BufferCell(cellname(6L), ByteBufferUtil.bytes("val6"), 1)); +// +// // verify insert. +// final SlicePredicate sp = new SlicePredicate(); +// sp.setSlice_range(new SliceRange()); +// sp.getSlice_range().setCount(100); +// sp.getSlice_range().setStart(ArrayUtils.EMPTY_BYTE_ARRAY); +// sp.getSlice_range().setFinish(ArrayUtils.EMPTY_BYTE_ARRAY); +// +// assertRowAndColCount(1, 6, false, cfs.getRangeSlice(Util.range("f", "g"), null, ThriftValidation.asIFilter(sp, cfs.metadata, scfName), 100)); +// +// // delete +// Mutation rm = new Mutation(keyspace.getName(), key.getKey()); +// rm.deleteRange(cfName, SuperColumns.startOf(scfName), SuperColumns.endOf(scfName), 2); +// rm.applyUnsafe(); +// +// // verify delete. +// assertRowAndColCount(1, 0, false, cfs.getRangeSlice(Util.range("f", "g"), null, ThriftValidation.asIFilter(sp, cfs.metadata, scfName), 100)); +// +// // flush +// cfs.forceBlockingFlush(); +// +// // re-verify delete. +// assertRowAndColCount(1, 0, false, cfs.getRangeSlice(Util.range("f", "g"), null, ThriftValidation.asIFilter(sp, cfs.metadata, scfName), 100)); +// +// // late insert. +// putColsSuper(cfs, key, scfName, +// new BufferCell(cellname(4L), ByteBufferUtil.bytes("val4"), 1L), +// new BufferCell(cellname(7L), ByteBufferUtil.bytes("val7"), 1L)); +// +// // re-verify delete. +// assertRowAndColCount(1, 0, false, cfs.getRangeSlice(Util.range("f", "g"), null, ThriftValidation.asIFilter(sp, cfs.metadata, scfName), 100)); +// +// // make sure new writes are recognized. +// putColsSuper(cfs, key, scfName, +// new BufferCell(cellname(3L), ByteBufferUtil.bytes("val3"), 3), +// new BufferCell(cellname(8L), ByteBufferUtil.bytes("val8"), 3), +// new BufferCell(cellname(9L), ByteBufferUtil.bytes("val9"), 3)); +// assertRowAndColCount(1, 3, false, cfs.getRangeSlice(Util.range("f", "g"), null, ThriftValidation.asIFilter(sp, cfs.metadata, scfName), 100)); +// } + +// private static void assertRowAndColCount(int rowCount, int colCount, boolean isDeleted, Collection rows) throws CharacterCodingException +// { +// assert rows.size() == rowCount : "rowcount " + rows.size(); +// for (Row row : rows) +// { +// assert row.cf != null : "cf was null"; +// assert row.cf.getColumnCount() == colCount : "colcount " + row.cf.getColumnCount() + "|" + str(row.cf); +// if (isDeleted) +// assert row.cf.isMarkedForDelete() : "cf not marked for delete"; +// } +// } +// +// private static String str(ColumnFamily cf) throws CharacterCodingException +// { +// StringBuilder sb = new StringBuilder(); +// for (Cell col : cf.getSortedColumns()) +// sb.append(String.format("(%s,%s,%d),", ByteBufferUtil.string(col.name().toByteBuffer()), ByteBufferUtil.string(col.value()), col.timestamp())); +// return sb.toString(); +// } @Test public void testDeleteStandardRowSticksAfterFlush() throws Throwable @@ -989,1292 +249,339 @@ public class ColumnFamilyStoreTest String cfName = CF_STANDARD1; Keyspace keyspace = Keyspace.open(keyspaceName); ColumnFamilyStore cfs = keyspace.getColumnFamilyStore(cfName); - DecoratedKey key = Util.dk("f-flush-resurrection"); - SlicePredicate sp = new SlicePredicate(); - sp.setSlice_range(new SliceRange()); - sp.getSlice_range().setCount(100); - sp.getSlice_range().setStart(ArrayUtils.EMPTY_BYTE_ARRAY); - sp.getSlice_range().setFinish(ArrayUtils.EMPTY_BYTE_ARRAY); + ByteBuffer col = ByteBufferUtil.bytes("val"); + ByteBuffer val = ByteBufferUtil.bytes("val1"); // insert - putColsStandard(cfs, key, column("col1", "val1", 1), column("col2", "val2", 1)); - assertRowAndColCount(1, 2, false, cfs.getRangeSlice(Util.range("f", "g"), null, ThriftValidation.asIFilter(sp, cfs.metadata, null), 100)); + ColumnDefinition newCol = new ColumnDefinition(cfs.metadata, ByteBufferUtil.bytes("val2"), AsciiType.instance, 1, ColumnDefinition.Kind.REGULAR); + new RowUpdateBuilder(cfs.metadata, 0, "key1").clustering("Column1").add("val", "val1").build().applyUnsafe(); + new RowUpdateBuilder(cfs.metadata, 0, "key2").clustering("Column1").add("val", "val1").build().applyUnsafe(); + assertRangeCount(cfs, col, val, 2); // flush. cfs.forceBlockingFlush(); // insert, don't flush - putColsStandard(cfs, key, column("col3", "val3", 1), column("col4", "val4", 1)); - assertRowAndColCount(1, 4, false, cfs.getRangeSlice(Util.range("f", "g"), null, ThriftValidation.asIFilter(sp, cfs.metadata, null), 100)); + new RowUpdateBuilder(cfs.metadata, 1, "key3").clustering("Column1").add("val", "val1").build().applyUnsafe(); + new RowUpdateBuilder(cfs.metadata, 1, "key4").clustering("Column1").add("val", "val1").build().applyUnsafe(); + assertRangeCount(cfs, col, val, 4); // delete (from sstable and memtable) - Mutation rm = new Mutation(keyspace.getName(), key.getKey()); - rm.delete(cfs.name, 2); - rm.applyUnsafe(); + RowUpdateBuilder.deleteRow(cfs.metadata, 5, "key1", "Column1").applyUnsafe(); + RowUpdateBuilder.deleteRow(cfs.metadata, 5, "key3", "Column1").applyUnsafe(); // verify delete - assertRowAndColCount(1, 0, true, cfs.getRangeSlice(Util.range("f", "g"), null, ThriftValidation.asIFilter(sp, cfs.metadata, null), 100)); + assertRangeCount(cfs, col, val, 2); // flush cfs.forceBlockingFlush(); // re-verify delete. // first breakage is right here because of CASSANDRA-1837. - assertRowAndColCount(1, 0, true, cfs.getRangeSlice(Util.range("f", "g"), null, ThriftValidation.asIFilter(sp, cfs.metadata, null), 100)); + assertRangeCount(cfs, col, val, 2); // simulate a 'late' insertion that gets put in after the deletion. should get inserted, but fail on read. - putColsStandard(cfs, key, column("col5", "val5", 1), column("col2", "val2", 1)); + new RowUpdateBuilder(cfs.metadata, 2, "key1").clustering("Column1").add("val", "val1").build().applyUnsafe(); + new RowUpdateBuilder(cfs.metadata, 2, "key3").clustering("Column1").add("val", "val1").build().applyUnsafe(); // should still be nothing there because we deleted this row. 2nd breakage, but was undetected because of 1837. - assertRowAndColCount(1, 0, true, cfs.getRangeSlice(Util.range("f", "g"), null, ThriftValidation.asIFilter(sp, cfs.metadata, null), 100)); + assertRangeCount(cfs, col, val, 2); // make sure that new writes are recognized. - putColsStandard(cfs, key, column("col6", "val6", 3), column("col7", "val7", 3)); - assertRowAndColCount(1, 2, true, cfs.getRangeSlice(Util.range("f", "g"), null, ThriftValidation.asIFilter(sp, cfs.metadata, null), 100)); + new RowUpdateBuilder(cfs.metadata, 10, "key5").clustering("Column1").add("val", "val1").build().applyUnsafe(); + new RowUpdateBuilder(cfs.metadata, 10, "key6").clustering("Column1").add("val", "val1").build().applyUnsafe(); + assertRangeCount(cfs, col, val, 4); // and it remains so after flush. (this wasn't failing before, but it's good to check.) cfs.forceBlockingFlush(); - assertRowAndColCount(1, 2, true, cfs.getRangeSlice(Util.range("f", "g"), null, ThriftValidation.asIFilter(sp, cfs.metadata, null), 100)); - } - - - private ColumnFamilyStore insertKey1Key2() - { - ColumnFamilyStore cfs = Keyspace.open(KEYSPACE2).getColumnFamilyStore(CF_STANDARD1); - List rms = new LinkedList<>(); - Mutation rm; - rm = new Mutation(KEYSPACE2, ByteBufferUtil.bytes("key1")); - rm.add(CF_STANDARD1, cellname("Column1"), ByteBufferUtil.bytes("asdf"), 0); - rms.add(rm); - Util.writeColumnFamily(rms); - - rm = new Mutation(KEYSPACE2, ByteBufferUtil.bytes("key2")); - rm.add(CF_STANDARD1, cellname("Column1"), ByteBufferUtil.bytes("asdf"), 0); - rms.add(rm); - return Util.writeColumnFamily(rms); + assertRangeCount(cfs, col, val, 4); } @Test public void testBackupAfterFlush() throws Throwable { - ColumnFamilyStore cfs = insertKey1Key2(); + ColumnFamilyStore cfs = Keyspace.open(KEYSPACE2).getColumnFamilyStore(CF_STANDARD1); + new RowUpdateBuilder(cfs.metadata, 0, ByteBufferUtil.bytes("key1")).clustering("Column1").add("val", "asdf").build().applyUnsafe(); + cfs.forceBlockingFlush(); + new RowUpdateBuilder(cfs.metadata, 0, ByteBufferUtil.bytes("key2")).clustering("Column1").add("val", "asdf").build().applyUnsafe(); + cfs.forceBlockingFlush(); for (int version = 1; version <= 2; ++version) { Descriptor existing = new Descriptor(cfs.directories.getDirectoryForNewSSTables(), KEYSPACE2, CF_STANDARD1, version, Descriptor.Type.FINAL); Descriptor desc = new Descriptor(Directories.getBackupsDirectory(existing), KEYSPACE2, CF_STANDARD1, version, Descriptor.Type.FINAL); for (Component c : new Component[]{ Component.DATA, Component.PRIMARY_INDEX, Component.FILTER, Component.STATS }) - assertTrue("can not find backedup file:" + desc.filenameFor(c), new File(desc.filenameFor(c)).exists()); + assertTrue("Cannot find backed-up file:" + desc.filenameFor(c), new File(desc.filenameFor(c)).exists()); } } - // CASSANDRA-3467. the key here is that supercolumn and subcolumn comparators are different - @Test - public void testSliceByNamesCommandOnUUIDTypeSCF() throws Throwable + // TODO: Fix once we have working supercolumns in 8099 +// // CASSANDRA-3467. the key here is that supercolumn and subcolumn comparators are different +// @Test +// public void testSliceByNamesCommandOnUUIDTypeSCF() throws Throwable +// { +// String keyspaceName = KEYSPACE1; +// String cfName = CF_SUPER6; +// ByteBuffer superColName = LexicalUUIDType.instance.fromString("a4ed3562-0e8e-4b41-bdfd-c45a2774682d"); +// Keyspace keyspace = Keyspace.open(keyspaceName); +// ColumnFamilyStore cfs = keyspace.getColumnFamilyStore(cfName); +// DecoratedKey key = Util.dk("slice-get-uuid-type"); +// +// // Insert a row with one supercolumn and multiple subcolumns +// putColsSuper(cfs, key, superColName, new BufferCell(cellname("a"), ByteBufferUtil.bytes("A"), 1), +// new BufferCell(cellname("b"), ByteBufferUtil.bytes("B"), 1)); +// +// // Get the entire supercolumn like normal +// ColumnFamily cfGet = cfs.getColumnFamily(QueryFilter.getIdentityFilter(key, cfName, System.currentTimeMillis())); +// assertEquals(ByteBufferUtil.bytes("A"), cfGet.getColumn(CellNames.compositeDense(superColName, ByteBufferUtil.bytes("a"))).value()); +// assertEquals(ByteBufferUtil.bytes("B"), cfGet.getColumn(CellNames.compositeDense(superColName, ByteBufferUtil.bytes("b"))).value()); +// +// // Now do the SliceByNamesCommand on the supercolumn, passing both subcolumns in as columns to get +// SortedSet sliceColNames = new TreeSet(cfs.metadata.comparator); +// sliceColNames.add(CellNames.compositeDense(superColName, ByteBufferUtil.bytes("a"))); +// sliceColNames.add(CellNames.compositeDense(superColName, ByteBufferUtil.bytes("b"))); +// SliceByNamesReadCommand cmd = new SliceByNamesReadCommand(keyspaceName, key.getKey(), cfName, System.currentTimeMillis(), new NamesQueryFilter(sliceColNames)); +// ColumnFamily cfSliced = cmd.getRow(keyspace).cf; +// +// // Make sure the slice returns the same as the straight get +// assertEquals(ByteBufferUtil.bytes("A"), cfSliced.getColumn(CellNames.compositeDense(superColName, ByteBufferUtil.bytes("a"))).value()); +// assertEquals(ByteBufferUtil.bytes("B"), cfSliced.getColumn(CellNames.compositeDense(superColName, ByteBufferUtil.bytes("b"))).value()); +// } + + // TODO: fix once SSTableSimpleWriter's back in +// @Test +// public void testRemoveUnfinishedCompactionLeftovers() throws Throwable +// { +// String ks = KEYSPACE1; +// String cf = CF_STANDARD3; // should be empty +// +// final CFMetaData cfmeta = Schema.instance.getCFMetaData(ks, cf); +// Directories dir = new Directories(cfmeta); +// ByteBuffer key = bytes("key"); +// +// // 1st sstable +// SSTableSimpleWriter writer = new SSTableSimpleWriter(dir.getDirectoryForNewSSTables(), cfmeta, StorageService.getPartitioner()); +// writer.newRow(key); +// writer.addColumn(bytes("col"), bytes("val"), 1); +// writer.close(); +// +// Map> sstables = dir.sstableLister().list(); +// assertEquals(1, sstables.size()); +// +// Map.Entry> sstableToOpen = sstables.entrySet().iterator().next(); +// final SSTableReader sstable1 = SSTableReader.open(sstableToOpen.getKey()); +// +// // simulate incomplete compaction +// writer = new SSTableSimpleWriter(dir.getDirectoryForNewSSTables(), +// cfmeta, StorageService.getPartitioner()) +// { +// protected SSTableWriter getWriter() +// { +// MetadataCollector collector = new MetadataCollector(cfmeta.comparator); +// collector.addAncestor(sstable1.descriptor.generation); // add ancestor from previously written sstable +// return SSTableWriter.create(createDescriptor(directory, metadata.ksName, metadata.cfName, DatabaseDescriptor.getSSTableFormat()), +// 0L, +// ActiveRepairService.UNREPAIRED_SSTABLE, +// metadata, +// DatabaseDescriptor.getPartitioner(), +// collector); +// } +// }; +// writer.newRow(key); +// writer.addColumn(bytes("col"), bytes("val"), 1); +// writer.close(); +// +// // should have 2 sstables now +// sstables = dir.sstableLister().list(); +// assertEquals(2, sstables.size()); +// +// SSTableReader sstable2 = SSTableReader.open(sstable1.descriptor); +// UUID compactionTaskID = SystemKeyspace.startCompaction( +// Keyspace.open(ks).getColumnFamilyStore(cf), +// Collections.singleton(sstable2)); +// +// Map unfinishedCompaction = new HashMap<>(); +// unfinishedCompaction.put(sstable1.descriptor.generation, compactionTaskID); +// ColumnFamilyStore.removeUnfinishedCompactionLeftovers(cfmeta, unfinishedCompaction); +// +// // 2nd sstable should be removed (only 1st sstable exists in set of size 1) +// sstables = dir.sstableLister().list(); +// assertEquals(1, sstables.size()); +// assertTrue(sstables.containsKey(sstable1.descriptor)); +// +// Map, Map> unfinished = SystemKeyspace.getUnfinishedCompactions(); +// assertTrue(unfinished.isEmpty()); +// sstable1.selfRef().release(); +// sstable2.selfRef().release(); +// } + + // TODO: Fix once SSTableSimpleWriter's back in + // @see CASSANDRA-6086 +// @Test +// public void testFailedToRemoveUnfinishedCompactionLeftovers() throws Throwable +// { +// final String ks = KEYSPACE1; +// final String cf = CF_STANDARD4; // should be empty +// +// final CFMetaData cfmeta = Schema.instance.getCFMetaData(ks, cf); +// Directories dir = new Directories(cfmeta); +// ByteBuffer key = bytes("key"); +// +// // Write SSTable generation 3 that has ancestors 1 and 2 +// final Set ancestors = Sets.newHashSet(1, 2); +// SSTableSimpleWriter writer = new SSTableSimpleWriter(dir.getDirectoryForNewSSTables(), +// cfmeta, StorageService.getPartitioner()) +// { +// protected SSTableWriter getWriter() +// { +// MetadataCollector collector = new MetadataCollector(cfmeta.comparator); +// for (int ancestor : ancestors) +// collector.addAncestor(ancestor); +// String file = new Descriptor(directory, ks, cf, 3, Descriptor.Type.TEMP).filenameFor(Component.DATA); +// return SSTableWriter.create(Descriptor.fromFilename(file), +// 0L, +// ActiveRepairService.UNREPAIRED_SSTABLE, +// metadata, +// StorageService.getPartitioner(), +// collector); +// } +// }; +// writer.newRow(key); +// writer.addColumn(bytes("col"), bytes("val"), 1); +// writer.close(); +// +// Map> sstables = dir.sstableLister().list(); +// assert sstables.size() == 1; +// +// Map.Entry> sstableToOpen = sstables.entrySet().iterator().next(); +// final SSTableReader sstable1 = SSTableReader.open(sstableToOpen.getKey()); +// +// // simulate we don't have generation in compaction_history +// Map unfinishedCompactions = new HashMap<>(); +// UUID compactionTaskID = UUID.randomUUID(); +// for (Integer ancestor : ancestors) +// unfinishedCompactions.put(ancestor, compactionTaskID); +// ColumnFamilyStore.removeUnfinishedCompactionLeftovers(cfmeta, unfinishedCompactions); +// +// // SSTable should not be deleted +// sstables = dir.sstableLister().list(); +// assert sstables.size() == 1; +// assert sstables.containsKey(sstable1.descriptor); +// } + + // TODO: Fix once SSTableSimpleWriter's back in +// @Test +// public void testLoadNewSSTablesAvoidsOverwrites() throws Throwable +// { +// String ks = KEYSPACE1; +// String cf = CF_STANDARD1; +// ColumnFamilyStore cfs = Keyspace.open(ks).getColumnFamilyStore(cf); +// SSTableDeletingTask.waitForDeletions(); +// +// final CFMetaData cfmeta = Schema.instance.getCFMetaData(ks, cf); +// Directories dir = new Directories(cfs.metadata); +// +// // clear old SSTables (probably left by CFS.clearUnsafe() calls in other tests) +// for (Map.Entry> entry : dir.sstableLister().list().entrySet()) +// { +// for (Component component : entry.getValue()) +// { +// FileUtils.delete(entry.getKey().filenameFor(component)); +// } +// } +// +// // sanity check +// int existingSSTables = dir.sstableLister().list().keySet().size(); +// assert existingSSTables == 0 : String.format("%d SSTables unexpectedly exist", existingSSTables); +// +// ByteBuffer key = bytes("key"); +// +// SSTableSimpleWriter writer = new SSTableSimpleWriter(dir.getDirectoryForNewSSTables(), +// cfmeta, StorageService.getPartitioner()) +// { +// @Override +// protected SSTableWriter getWriter() +// { +// // hack for reset generation +// generation.set(0); +// return super.getWriter(); +// } +// }; +// writer.newRow(key); +// writer.addColumn(bytes("col"), bytes("val"), 1); +// writer.close(); +// +// writer = new SSTableSimpleWriter(dir.getDirectoryForNewSSTables(), +// cfmeta, StorageService.getPartitioner()); +// writer.newRow(key); +// writer.addColumn(bytes("col"), bytes("val"), 1); +// writer.close(); +// +// Set generations = new HashSet<>(); +// for (Descriptor descriptor : dir.sstableLister().list().keySet()) +// generations.add(descriptor.generation); +// +// // we should have two generations: [1, 2] +// assertEquals(2, generations.size()); +// assertTrue(generations.contains(1)); +// assertTrue(generations.contains(2)); +// +// assertEquals(0, cfs.getSSTables().size()); +// +// // start the generation counter at 1 again (other tests have incremented it already) +// cfs.resetFileIndexGenerator(); +// +// boolean incrementalBackupsEnabled = DatabaseDescriptor.isIncrementalBackupsEnabled(); +// try +// { +// // avoid duplicate hardlinks to incremental backups +// DatabaseDescriptor.setIncrementalBackupsEnabled(false); +// cfs.loadNewSSTables(); +// } +// finally +// { +// DatabaseDescriptor.setIncrementalBackupsEnabled(incrementalBackupsEnabled); +// } +// +// assertEquals(2, cfs.getSSTables().size()); +// generations = new HashSet<>(); +// for (Descriptor descriptor : dir.sstableLister().list().keySet()) +// generations.add(descriptor.generation); +// +// // normally they would get renamed to generations 1 and 2, but since those filenames already exist, +// // they get skipped and we end up with generations 3 and 4 +// assertEquals(2, generations.size()); +// assertTrue(generations.contains(3)); +// assertTrue(generations.contains(4)); +// } + + public void reTest(ColumnFamilyStore cfs, Runnable verify) throws Exception { - String keyspaceName = KEYSPACE1; - String cfName = CF_SUPER6; - ByteBuffer superColName = LexicalUUIDType.instance.fromString("a4ed3562-0e8e-4b41-bdfd-c45a2774682d"); - Keyspace keyspace = Keyspace.open(keyspaceName); - ColumnFamilyStore cfs = keyspace.getColumnFamilyStore(cfName); - DecoratedKey key = Util.dk("slice-get-uuid-type"); - - // Insert a row with one supercolumn and multiple subcolumns - putColsSuper(cfs, key, superColName, new BufferCell(cellname("a"), ByteBufferUtil.bytes("A"), 1), - new BufferCell(cellname("b"), ByteBufferUtil.bytes("B"), 1)); - - // Get the entire supercolumn like normal - ColumnFamily cfGet = cfs.getColumnFamily(QueryFilter.getIdentityFilter(key, cfName, System.currentTimeMillis())); - assertEquals(ByteBufferUtil.bytes("A"), cfGet.getColumn(CellNames.compositeDense(superColName, ByteBufferUtil.bytes("a"))).value()); - assertEquals(ByteBufferUtil.bytes("B"), cfGet.getColumn(CellNames.compositeDense(superColName, ByteBufferUtil.bytes("b"))).value()); - - // Now do the SliceByNamesCommand on the supercolumn, passing both subcolumns in as columns to get - SortedSet sliceColNames = new TreeSet(cfs.metadata.comparator); - sliceColNames.add(CellNames.compositeDense(superColName, ByteBufferUtil.bytes("a"))); - sliceColNames.add(CellNames.compositeDense(superColName, ByteBufferUtil.bytes("b"))); - SliceByNamesReadCommand cmd = new SliceByNamesReadCommand(keyspaceName, key.getKey(), cfName, System.currentTimeMillis(), new NamesQueryFilter(sliceColNames)); - ColumnFamily cfSliced = cmd.getRow(keyspace).cf; - - // Make sure the slice returns the same as the straight get - assertEquals(ByteBufferUtil.bytes("A"), cfSliced.getColumn(CellNames.compositeDense(superColName, ByteBufferUtil.bytes("a"))).value()); - assertEquals(ByteBufferUtil.bytes("B"), cfSliced.getColumn(CellNames.compositeDense(superColName, ByteBufferUtil.bytes("b"))).value()); - } - - @Test - public void testSliceByNamesCommandOldMetadata() throws Throwable - { - String keyspaceName = KEYSPACE1; - String cfName= CF_STANDARD1; - DecoratedKey key = Util.dk("slice-name-old-metadata"); - CellName cname = cellname("c1"); - Keyspace keyspace = Keyspace.open(keyspaceName); - ColumnFamilyStore cfs = keyspace.getColumnFamilyStore(cfName); - cfs.truncateBlocking(); - - // Create a cell a 'high timestamp' - putColsStandard(cfs, key, new BufferCell(cname, ByteBufferUtil.bytes("a"), 2)); + verify.run(); cfs.forceBlockingFlush(); - - // Nuke the metadata and reload that sstable - Collection ssTables = cfs.getSSTables(); - assertEquals(1, ssTables.size()); - cfs.clearUnsafe(); - assertEquals(0, cfs.getSSTables().size()); - - new File(ssTables.iterator().next().descriptor.filenameFor(Component.STATS)).delete(); - cfs.loadNewSSTables(); - - // Add another cell with a lower timestamp - putColsStandard(cfs, key, new BufferCell(cname, ByteBufferUtil.bytes("b"), 1)); - - // Test fetching the cell by name returns the first cell - SliceByNamesReadCommand cmd = new SliceByNamesReadCommand(keyspaceName, key.getKey(), cfName, System.currentTimeMillis(), new NamesQueryFilter(FBUtilities.singleton(cname, cfs.getComparator()))); - ColumnFamily cf = cmd.getRow(keyspace).cf; - Cell cell = cf.getColumn(cname); - assert cell.value().equals(ByteBufferUtil.bytes("a")) : "expecting a, got " + ByteBufferUtil.string(cell.value()); - - Keyspace.clear(KEYSPACE1); // CASSANDRA-7195 + verify.run(); } - private static void assertTotalColCount(Collection rows, int expectedCount) + private void assertRangeCount(ColumnFamilyStore cfs, ByteBuffer col, ByteBuffer val, int count) { - int columns = 0; - for (Row row : rows) + assertRangeCount(cfs, cfs.metadata.getColumnDefinition(col), val, count); + } + + private void assertRangeCount(ColumnFamilyStore cfs, ColumnDefinition col, ByteBuffer val, int count) + { + + int found = 0; + if (count != 0) { - columns += row.getLiveCount(new SliceQueryFilter(ColumnSlice.ALL_COLUMNS_ARRAY, false, expectedCount), System.currentTimeMillis()); - } - assert columns == expectedCount : "Expected " + expectedCount + " live columns but got " + columns + ": " + rows; - } - - - @Test - public void testRangeSliceColumnsLimit() throws Throwable - { - String keyspaceName = KEYSPACE1; - String cfName = CF_STANDARD1; - Keyspace keyspace = Keyspace.open(keyspaceName); - ColumnFamilyStore cfs = keyspace.getColumnFamilyStore(cfName); - cfs.clearUnsafe(); - - Cell[] cols = new Cell[5]; - for (int i = 0; i < 5; i++) - cols[i] = column("c" + i, "value", 1); - - putColsStandard(cfs, Util.dk("a"), cols[0], cols[1], cols[2], cols[3], cols[4]); - putColsStandard(cfs, Util.dk("b"), cols[0], cols[1]); - putColsStandard(cfs, Util.dk("c"), cols[0], cols[1], cols[2], cols[3]); - cfs.forceBlockingFlush(); - - SlicePredicate sp = new SlicePredicate(); - sp.setSlice_range(new SliceRange()); - sp.getSlice_range().setCount(1); - sp.getSlice_range().setStart(ArrayUtils.EMPTY_BYTE_ARRAY); - sp.getSlice_range().setFinish(ArrayUtils.EMPTY_BYTE_ARRAY); - - assertTotalColCount(cfs.getRangeSlice(Util.range("", ""), - null, - ThriftValidation.asIFilter(sp, cfs.metadata, null), - 3, - System.currentTimeMillis(), - true, - false), - 3); - assertTotalColCount(cfs.getRangeSlice(Util.range("", ""), - null, - ThriftValidation.asIFilter(sp, cfs.metadata, null), - 5, - System.currentTimeMillis(), - true, - false), - 5); - assertTotalColCount(cfs.getRangeSlice(Util.range("", ""), - null, - ThriftValidation.asIFilter(sp, cfs.metadata, null), - 8, - System.currentTimeMillis(), - true, - false), - 8); - assertTotalColCount(cfs.getRangeSlice(Util.range("", ""), - null, - ThriftValidation.asIFilter(sp, cfs.metadata, null), - 10, - System.currentTimeMillis(), - true, - false), - 10); - assertTotalColCount(cfs.getRangeSlice(Util.range("", ""), - null, - ThriftValidation.asIFilter(sp, cfs.metadata, null), - 100, - System.currentTimeMillis(), - true, - false), - 11); - - // Check that when querying by name, we always include all names for a - // gien row even if it means returning more columns than requested (this is necesseray for CQL) - sp = new SlicePredicate(); - sp.setColumn_names(Arrays.asList( - ByteBufferUtil.bytes("c0"), - ByteBufferUtil.bytes("c1"), - ByteBufferUtil.bytes("c2") - )); - - assertTotalColCount(cfs.getRangeSlice(Util.range("", ""), - null, - ThriftValidation.asIFilter(sp, cfs.metadata, null), - 1, - System.currentTimeMillis(), - true, - false), - 3); - assertTotalColCount(cfs.getRangeSlice(Util.range("", ""), - null, - ThriftValidation.asIFilter(sp, cfs.metadata, null), - 4, - System.currentTimeMillis(), - true, - false), - 5); - assertTotalColCount(cfs.getRangeSlice(Util.range("", ""), - null, - ThriftValidation.asIFilter(sp, cfs.metadata, null), - 5, - System.currentTimeMillis(), - true, - false), - 5); - assertTotalColCount(cfs.getRangeSlice(Util.range("", ""), - null, - ThriftValidation.asIFilter(sp, cfs.metadata, null), - 6, - System.currentTimeMillis(), - true, - false), - 8); - assertTotalColCount(cfs.getRangeSlice(Util.range("", ""), - null, - ThriftValidation.asIFilter(sp, cfs.metadata, null), - 100, - System.currentTimeMillis(), - true, - false), - 8); - } - - @Test - public void testRangeSlicePaging() throws Throwable - { - String keyspaceName = KEYSPACE1; - String cfName = CF_STANDARD1; - Keyspace keyspace = Keyspace.open(keyspaceName); - ColumnFamilyStore cfs = keyspace.getColumnFamilyStore(cfName); - cfs.clearUnsafe(); - - Cell[] cols = new Cell[4]; - for (int i = 0; i < 4; i++) - cols[i] = column("c" + i, "value", 1); - - DecoratedKey ka = Util.dk("a"); - DecoratedKey kb = Util.dk("b"); - DecoratedKey kc = Util.dk("c"); - - RowPosition min = Util.rp(""); - - putColsStandard(cfs, ka, cols[0], cols[1], cols[2], cols[3]); - putColsStandard(cfs, kb, cols[0], cols[1], cols[2]); - putColsStandard(cfs, kc, cols[0], cols[1], cols[2], cols[3]); - cfs.forceBlockingFlush(); - - SlicePredicate sp = new SlicePredicate(); - sp.setSlice_range(new SliceRange()); - sp.getSlice_range().setCount(1); - sp.getSlice_range().setStart(ArrayUtils.EMPTY_BYTE_ARRAY); - sp.getSlice_range().setFinish(ArrayUtils.EMPTY_BYTE_ARRAY); - - Collection rows; - Row row, row1, row2; - IDiskAtomFilter filter = ThriftValidation.asIFilter(sp, cfs.metadata, null); - - rows = cfs.getRangeSlice(cfs.makeExtendedFilter(Util.range("", ""), filter, null, 3, true, true, System.currentTimeMillis())); - assert rows.size() == 1 : "Expected 1 row, got " + toString(rows); - row = rows.iterator().next(); - assertColumnNames(row, "c0", "c1", "c2"); - - sp.getSlice_range().setStart(ByteBufferUtil.getArray(ByteBufferUtil.bytes("c2"))); - filter = ThriftValidation.asIFilter(sp, cfs.metadata, null); - rows = cfs.getRangeSlice(cfs.makeExtendedFilter(new Bounds(ka, min), filter, null, 3, true, true, System.currentTimeMillis())); - assert rows.size() == 2 : "Expected 2 rows, got " + toString(rows); - Iterator iter = rows.iterator(); - row1 = iter.next(); - row2 = iter.next(); - assertColumnNames(row1, "c2", "c3"); - assertColumnNames(row2, "c0"); - - sp.getSlice_range().setStart(ByteBufferUtil.getArray(ByteBufferUtil.bytes("c0"))); - filter = ThriftValidation.asIFilter(sp, cfs.metadata, null); - rows = cfs.getRangeSlice(cfs.makeExtendedFilter(new Bounds(row2.key, min), filter, null, 3, true, true, System.currentTimeMillis())); - assert rows.size() == 1 : "Expected 1 row, got " + toString(rows); - row = rows.iterator().next(); - assertColumnNames(row, "c0", "c1", "c2"); - - sp.getSlice_range().setStart(ByteBufferUtil.getArray(ByteBufferUtil.bytes("c2"))); - filter = ThriftValidation.asIFilter(sp, cfs.metadata, null); - rows = cfs.getRangeSlice(cfs.makeExtendedFilter(new Bounds(row.key, min), filter, null, 3, true, true, System.currentTimeMillis())); - assert rows.size() == 2 : "Expected 2 rows, got " + toString(rows); - iter = rows.iterator(); - row1 = iter.next(); - row2 = iter.next(); - assertColumnNames(row1, "c2"); - assertColumnNames(row2, "c0", "c1"); - - // Paging within bounds - SliceQueryFilter sf = new SliceQueryFilter(cellname("c1"), - cellname("c2"), - false, - 0); - rows = cfs.getRangeSlice(cfs.makeExtendedFilter(new Bounds(ka, kc), sf, cellname("c2"), cellname("c1"), null, 2, true, System.currentTimeMillis())); - assert rows.size() == 2 : "Expected 2 rows, got " + toString(rows); - iter = rows.iterator(); - row1 = iter.next(); - row2 = iter.next(); - assertColumnNames(row1, "c2"); - assertColumnNames(row2, "c1"); - - rows = cfs.getRangeSlice(cfs.makeExtendedFilter(new Bounds(kb, kc), sf, cellname("c1"), cellname("c1"), null, 10, true, System.currentTimeMillis())); - assert rows.size() == 2 : "Expected 2 rows, got " + toString(rows); - iter = rows.iterator(); - row1 = iter.next(); - row2 = iter.next(); - assertColumnNames(row1, "c1", "c2"); - assertColumnNames(row2, "c1"); - } - - private static String toString(Collection rows) - { - try - { - StringBuilder sb = new StringBuilder(); - for (Row row : rows) + for (FilteredPartition partition : Util.getAll(Util.cmd(cfs).filterOn(col.name.toString(), Operator.EQ, val).build())) { - sb.append("{"); - sb.append(ByteBufferUtil.string(row.key.getKey())); - sb.append(":"); - if (row.cf != null && !row.cf.isEmpty()) + for (Row r : partition) { - for (Cell c : row.cf) - sb.append(" ").append(row.cf.getComparator().getString(c.name())); + if (r.getCell(col).value().equals(val)) + ++found; } - sb.append("} "); - } - return sb.toString(); - } - catch (Exception e) - { - throw new RuntimeException(e); - } - } - - private static void assertColumnNames(Row row, String ... columnNames) throws Exception - { - if (row == null || row.cf == null) - throw new AssertionError("The row should not be empty"); - - Iterator columns = row.cf.getSortedColumns().iterator(); - Iterator names = Arrays.asList(columnNames).iterator(); - - while (columns.hasNext()) - { - Cell c = columns.next(); - assert names.hasNext() : "Got more columns that expected (first unexpected column: " + ByteBufferUtil.string(c.name().toByteBuffer()) + ")"; - String n = names.next(); - assert c.name().toByteBuffer().equals(ByteBufferUtil.bytes(n)) : "Expected " + n + ", got " + ByteBufferUtil.string(c.name().toByteBuffer()); - } - assert !names.hasNext() : "Missing expected column " + names.next(); - } - - private static DecoratedKey idk(int i) - { - return Util.dk(String.valueOf(i)); - } - - @Test - public void testRangeSliceInclusionExclusion() throws Throwable - { - String keyspaceName = KEYSPACE1; - String cfName = CF_STANDARD1; - Keyspace keyspace = Keyspace.open(keyspaceName); - ColumnFamilyStore cfs = keyspace.getColumnFamilyStore(cfName); - cfs.clearUnsafe(); - - Cell[] cols = new Cell[5]; - for (int i = 0; i < 5; i++) - cols[i] = column("c" + i, "value", 1); - - for (int i = 0; i <= 9; i++) - { - putColsStandard(cfs, idk(i), column("name", "value", 1)); - } - cfs.forceBlockingFlush(); - - SlicePredicate sp = new SlicePredicate(); - sp.setSlice_range(new SliceRange()); - sp.getSlice_range().setCount(1); - sp.getSlice_range().setStart(ArrayUtils.EMPTY_BYTE_ARRAY); - sp.getSlice_range().setFinish(ArrayUtils.EMPTY_BYTE_ARRAY); - IDiskAtomFilter qf = ThriftValidation.asIFilter(sp, cfs.metadata, null); - - List rows; - - // Start and end inclusive - rows = cfs.getRangeSlice(new Bounds(rp("2"), rp("7")), null, qf, 100); - assert rows.size() == 6; - assert rows.get(0).key.equals(idk(2)); - assert rows.get(rows.size() - 1).key.equals(idk(7)); - - // Start and end excluded - rows = cfs.getRangeSlice(new ExcludingBounds(rp("2"), rp("7")), null, qf, 100); - assert rows.size() == 4; - assert rows.get(0).key.equals(idk(3)); - assert rows.get(rows.size() - 1).key.equals(idk(6)); - - // Start excluded, end included - rows = cfs.getRangeSlice(new Range(rp("2"), rp("7")), null, qf, 100); - assert rows.size() == 5; - assert rows.get(0).key.equals(idk(3)); - assert rows.get(rows.size() - 1).key.equals(idk(7)); - - // Start included, end excluded - rows = cfs.getRangeSlice(new IncludingExcludingBounds(rp("2"), rp("7")), null, qf, 100); - assert rows.size() == 5; - assert rows.get(0).key.equals(idk(2)); - assert rows.get(rows.size() - 1).key.equals(idk(6)); - } - - @Test - public void testKeysSearcher() throws Exception - { - // Create secondary index and flush to disk - Keyspace keyspace = Keyspace.open(KEYSPACE1); - ColumnFamilyStore store = keyspace.getColumnFamilyStore(CF_INDEX1); - - store.truncateBlocking(); - - for (int i = 0; i < 10; i++) - { - ByteBuffer key = ByteBufferUtil.bytes(String.valueOf("k" + i)); - Mutation rm = new Mutation(KEYSPACE1, key); - rm.add(CF_INDEX1, cellname("birthdate"), LongType.instance.decompose(1L), System.currentTimeMillis()); - rm.applyUnsafe(); - } - - store.forceBlockingFlush(); - - IndexExpression expr = new IndexExpression(ByteBufferUtil.bytes("birthdate"), Operator.EQ, LongType.instance.decompose(1L)); - // explicitly tell to the KeysSearcher to use column limiting for rowsPerQuery to trigger bogus columnsRead--; (CASSANDRA-3996) - List rows = store.search(store.makeExtendedFilter(Util.range("", ""), new IdentityQueryFilter(), Arrays.asList(expr), 10, true, false, System.currentTimeMillis())); - - assert rows.size() == 10; - } - - @SuppressWarnings("unchecked") - @Test - public void testMultiRangeSomeEmptyNoIndex() throws Throwable - { - // in order not to change thrift interfaces at this stage we build SliceQueryFilter - // directly instead of using QueryFilter to build it for us - ColumnSlice[] ranges = new ColumnSlice[] { - new ColumnSlice(Composites.EMPTY, cellname("colA")), - new ColumnSlice(cellname("colC"), cellname("colE")), - new ColumnSlice(cellname("colF"), cellname("colF")), - new ColumnSlice(cellname("colG"), cellname("colG")), - new ColumnSlice(cellname("colI"), Composites.EMPTY) }; - - ColumnSlice[] rangesReversed = new ColumnSlice[] { - new ColumnSlice(Composites.EMPTY, cellname("colI")), - new ColumnSlice(cellname("colG"), cellname("colG")), - new ColumnSlice(cellname("colF"), cellname("colF")), - new ColumnSlice(cellname("colE"), cellname("colC")), - new ColumnSlice(cellname("colA"), Composites.EMPTY) }; - - String tableName = KEYSPACE1; - String cfName = CF_STANDARD1; - Keyspace table = Keyspace.open(tableName); - ColumnFamilyStore cfs = table.getColumnFamilyStore(cfName); - cfs.clearUnsafe(); - - String[] letters = new String[] { "a", "b", "c", "d", "i" }; - Cell[] cols = new Cell[letters.length]; - for (int i = 0; i < cols.length; i++) - { - cols[i] = new BufferCell(cellname("col" + letters[i].toUpperCase()), - ByteBuffer.wrap(new byte[1]), 1); - } - - putColsStandard(cfs, dk("a"), cols); - - cfs.forceBlockingFlush(); - - SliceQueryFilter multiRangeForward = new SliceQueryFilter(ranges, false, 100); - SliceQueryFilter multiRangeForwardWithCounting = new SliceQueryFilter(ranges, false, 3); - SliceQueryFilter multiRangeReverse = new SliceQueryFilter(rangesReversed, true, 100); - SliceQueryFilter multiRangeReverseWithCounting = new SliceQueryFilter(rangesReversed, true, 3); - - findRowGetSlicesAndAssertColsFound(cfs, multiRangeForward, "a", "colA", "colC", "colD", "colI"); - findRowGetSlicesAndAssertColsFound(cfs, multiRangeForwardWithCounting, "a", "colA", "colC", "colD"); - findRowGetSlicesAndAssertColsFound(cfs, multiRangeReverse, "a", "colI", "colD", "colC", "colA"); - findRowGetSlicesAndAssertColsFound(cfs, multiRangeReverseWithCounting, "a", "colI", "colD", "colC"); - } - - @SuppressWarnings("unchecked") - @Test - public void testMultiRangeSomeEmptyIndexed() throws Throwable - { - // in order not to change thrift interfaces at this stage we build SliceQueryFilter - // directly instead of using QueryFilter to build it for us - ColumnSlice[] ranges = new ColumnSlice[] { - new ColumnSlice(Composites.EMPTY, cellname("colA")), - new ColumnSlice(cellname("colC"), cellname("colE")), - new ColumnSlice(cellname("colF"), cellname("colF")), - new ColumnSlice(cellname("colG"), cellname("colG")), - new ColumnSlice(cellname("colI"), Composites.EMPTY) }; - - ColumnSlice[] rangesReversed = new ColumnSlice[] { - new ColumnSlice(Composites.EMPTY, cellname("colI")), - new ColumnSlice(cellname("colG"), cellname("colG")), - new ColumnSlice(cellname("colF"), cellname("colF")), - new ColumnSlice(cellname("colE"), cellname("colC")), - new ColumnSlice(cellname("colA"), Composites.EMPTY) }; - - String tableName = KEYSPACE1; - String cfName = CF_STANDARD1; - Keyspace table = Keyspace.open(tableName); - ColumnFamilyStore cfs = table.getColumnFamilyStore(cfName); - cfs.clearUnsafe(); - - String[] letters = new String[] { "a", "b", "c", "d", "i" }; - Cell[] cols = new Cell[letters.length]; - for (int i = 0; i < cols.length; i++) - { - cols[i] = new BufferCell(cellname("col" + letters[i].toUpperCase()), - ByteBuffer.wrap(new byte[1366]), 1); - } - - putColsStandard(cfs, dk("a"), cols); - - cfs.forceBlockingFlush(); - - SliceQueryFilter multiRangeForward = new SliceQueryFilter(ranges, false, 100); - SliceQueryFilter multiRangeForwardWithCounting = new SliceQueryFilter(ranges, false, 3); - SliceQueryFilter multiRangeReverse = new SliceQueryFilter(rangesReversed, true, 100); - SliceQueryFilter multiRangeReverseWithCounting = new SliceQueryFilter(rangesReversed, true, 3); - - findRowGetSlicesAndAssertColsFound(cfs, multiRangeForward, "a", "colA", "colC", "colD", "colI"); - findRowGetSlicesAndAssertColsFound(cfs, multiRangeForwardWithCounting, "a", "colA", "colC", "colD"); - findRowGetSlicesAndAssertColsFound(cfs, multiRangeReverse, "a", "colI", "colD", "colC", "colA"); - findRowGetSlicesAndAssertColsFound(cfs, multiRangeReverseWithCounting, "a", "colI", "colD", "colC"); - } - - @SuppressWarnings("unchecked") - @Test - public void testMultiRangeContiguousNoIndex() throws Throwable - { - // in order not to change thrift interfaces at this stage we build SliceQueryFilter - // directly instead of using QueryFilter to build it for us - ColumnSlice[] ranges = new ColumnSlice[] { - new ColumnSlice(Composites.EMPTY, cellname("colA")), - new ColumnSlice(cellname("colC"), cellname("colE")), - new ColumnSlice(cellname("colF"), cellname("colF")), - new ColumnSlice(cellname("colG"), cellname("colG")), - new ColumnSlice(cellname("colI"), Composites.EMPTY) }; - - ColumnSlice[] rangesReversed = new ColumnSlice[] { - new ColumnSlice(Composites.EMPTY, cellname("colI")), - new ColumnSlice(cellname("colG"), cellname("colG")), - new ColumnSlice(cellname("colF"), cellname("colF")), - new ColumnSlice(cellname("colE"), cellname("colC")), - new ColumnSlice(cellname("colA"), Composites.EMPTY) }; - - String tableName = KEYSPACE1; - String cfName = CF_STANDARD1; - Keyspace table = Keyspace.open(tableName); - ColumnFamilyStore cfs = table.getColumnFamilyStore(cfName); - cfs.clearUnsafe(); - - String[] letters = new String[] { "a", "b", "c", "d", "e", "f", "g", "h", "i" }; - Cell[] cols = new Cell[letters.length]; - for (int i = 0; i < cols.length; i++) - { - cols[i] = new BufferCell(cellname("col" + letters[i].toUpperCase()), - ByteBuffer.wrap(new byte[1]), 1); - } - - putColsStandard(cfs, dk("a"), cols); - - cfs.forceBlockingFlush(); - - SliceQueryFilter multiRangeForward = new SliceQueryFilter(ranges, false, 100); - SliceQueryFilter multiRangeForwardWithCounting = new SliceQueryFilter(ranges, false, 3); - SliceQueryFilter multiRangeReverse = new SliceQueryFilter(rangesReversed, true, 100); - SliceQueryFilter multiRangeReverseWithCounting = new SliceQueryFilter(rangesReversed, true, 3); - - findRowGetSlicesAndAssertColsFound(cfs, multiRangeForward, "a", "colA", "colC", "colD", "colE", "colF", "colG", "colI"); - findRowGetSlicesAndAssertColsFound(cfs, multiRangeForwardWithCounting, "a", "colA", "colC", "colD"); - findRowGetSlicesAndAssertColsFound(cfs, multiRangeReverse, "a", "colI", "colG", "colF", "colE", "colD", "colC", "colA"); - findRowGetSlicesAndAssertColsFound(cfs, multiRangeReverseWithCounting, "a", "colI", "colG", "colF"); - - } - - @SuppressWarnings("unchecked") - @Test - public void testMultiRangeContiguousIndexed() throws Throwable - { - // in order not to change thrift interfaces at this stage we build SliceQueryFilter - // directly instead of using QueryFilter to build it for us - ColumnSlice[] ranges = new ColumnSlice[] { - new ColumnSlice(Composites.EMPTY, cellname("colA")), - new ColumnSlice(cellname("colC"), cellname("colE")), - new ColumnSlice(cellname("colF"), cellname("colF")), - new ColumnSlice(cellname("colG"), cellname("colG")), - new ColumnSlice(cellname("colI"), Composites.EMPTY) }; - - ColumnSlice[] rangesReversed = new ColumnSlice[] { - new ColumnSlice(Composites.EMPTY, cellname("colI")), - new ColumnSlice(cellname("colG"), cellname("colG")), - new ColumnSlice(cellname("colF"), cellname("colF")), - new ColumnSlice(cellname("colE"), cellname("colC")), - new ColumnSlice(cellname("colA"), Composites.EMPTY) }; - - String tableName = KEYSPACE1; - String cfName = CF_STANDARD1; - Keyspace table = Keyspace.open(tableName); - ColumnFamilyStore cfs = table.getColumnFamilyStore(cfName); - cfs.clearUnsafe(); - - String[] letters = new String[] { "a", "b", "c", "d", "e", "f", "g", "h", "i" }; - Cell[] cols = new Cell[letters.length]; - for (int i = 0; i < cols.length; i++) - { - cols[i] = new BufferCell(cellname("col" + letters[i].toUpperCase()), - ByteBuffer.wrap(new byte[1366]), 1); - } - - putColsStandard(cfs, dk("a"), cols); - - cfs.forceBlockingFlush(); - - SliceQueryFilter multiRangeForward = new SliceQueryFilter(ranges, false, 100); - SliceQueryFilter multiRangeForwardWithCounting = new SliceQueryFilter(ranges, false, 3); - SliceQueryFilter multiRangeReverse = new SliceQueryFilter(rangesReversed, true, 100); - SliceQueryFilter multiRangeReverseWithCounting = new SliceQueryFilter(rangesReversed, true, 3); - - findRowGetSlicesAndAssertColsFound(cfs, multiRangeForward, "a", "colA", "colC", "colD", "colE", "colF", "colG", "colI"); - findRowGetSlicesAndAssertColsFound(cfs, multiRangeForwardWithCounting, "a", "colA", "colC", "colD"); - findRowGetSlicesAndAssertColsFound(cfs, multiRangeReverse, "a", "colI", "colG", "colF", "colE", "colD", "colC", "colA"); - findRowGetSlicesAndAssertColsFound(cfs, multiRangeReverseWithCounting, "a", "colI", "colG", "colF"); - - } - - @SuppressWarnings("unchecked") - @Test - public void testMultiRangeIndexed() throws Throwable - { - // in order not to change thrift interfaces at this stage we build SliceQueryFilter - // directly instead of using QueryFilter to build it for us - ColumnSlice[] ranges = new ColumnSlice[] { - new ColumnSlice(Composites.EMPTY, cellname("colA")), - new ColumnSlice(cellname("colC"), cellname("colE")), - new ColumnSlice(cellname("colG"), cellname("colG")), - new ColumnSlice(cellname("colI"), Composites.EMPTY) }; - - ColumnSlice[] rangesReversed = new ColumnSlice[] { - new ColumnSlice(Composites.EMPTY, cellname("colI")), - new ColumnSlice(cellname("colG"), cellname("colG")), - new ColumnSlice(cellname("colE"), cellname("colC")), - new ColumnSlice(cellname("colA"), Composites.EMPTY) }; - - String keyspaceName = KEYSPACE1; - String cfName = CF_STANDARD1; - Keyspace keyspace = Keyspace.open(keyspaceName); - ColumnFamilyStore cfs = keyspace.getColumnFamilyStore(cfName); - cfs.clearUnsafe(); - - String[] letters = new String[] { "a", "b", "c", "d", "e", "f", "g", "h", "i" }; - Cell[] cols = new Cell[letters.length]; - for (int i = 0; i < cols.length; i++) - { - cols[i] = new BufferCell(cellname("col" + letters[i].toUpperCase()), - // use 1366 so that three cols make an index segment - ByteBuffer.wrap(new byte[1366]), 1); - } - - putColsStandard(cfs, dk("a"), cols); - - cfs.forceBlockingFlush(); - - // this setup should generate the following row (assuming indexes are of 4Kb each): - // [colA, colB, colC, colD, colE, colF, colG, colH, colI] - // indexed as: - // index0 [colA, colC] - // index1 [colD, colF] - // index2 [colG, colI] - // and we're looking for the ranges: - // range0 [____, colA] - // range1 [colC, colE] - // range2 [colG, ColG] - // range3 [colI, ____] - - SliceQueryFilter multiRangeForward = new SliceQueryFilter(ranges, false, 100); - SliceQueryFilter multiRangeForwardWithCounting = new SliceQueryFilter(ranges, false, 3); - SliceQueryFilter multiRangeReverse = new SliceQueryFilter(rangesReversed, true, 100); - SliceQueryFilter multiRangeReverseWithCounting = new SliceQueryFilter(rangesReversed, true, 3); - - findRowGetSlicesAndAssertColsFound(cfs, multiRangeForward, "a", "colA", "colC", "colD", "colE", "colG", "colI"); - findRowGetSlicesAndAssertColsFound(cfs, multiRangeForwardWithCounting, "a", "colA", "colC", "colD"); - findRowGetSlicesAndAssertColsFound(cfs, multiRangeReverse, "a", "colI", "colG", "colE", "colD", "colC", "colA"); - findRowGetSlicesAndAssertColsFound(cfs, multiRangeReverseWithCounting, "a", "colI", "colG", "colE"); - - } - - @Test - public void testMultipleRangesSlicesNoIndexedColumns() throws Throwable - { - // small values so that cols won't be indexed - testMultiRangeSlicesBehavior(prepareMultiRangeSlicesTest(10, true)); - } - - @Test - public void testMultipleRangesSlicesWithIndexedColumns() throws Throwable - { - // min val size before cols are indexed is 4kb while testing so lets make sure cols are indexed - testMultiRangeSlicesBehavior(prepareMultiRangeSlicesTest(1024, true)); - } - - @Test - public void testMultipleRangesSlicesInMemory() throws Throwable - { - // small values so that cols won't be indexed - testMultiRangeSlicesBehavior(prepareMultiRangeSlicesTest(10, false)); - } - - @Test - public void testRemoveUnfinishedCompactionLeftovers() throws Throwable - { - String ks = KEYSPACE1; - String cf = CF_STANDARD3; // should be empty - - final CFMetaData cfmeta = Schema.instance.getCFMetaData(ks, cf); - Directories dir = new Directories(cfmeta); - ByteBuffer key = bytes("key"); - - // 1st sstable - SSTableSimpleWriter writer = new SSTableSimpleWriter(dir.getDirectoryForNewSSTables(), cfmeta, StorageService.getPartitioner()); - writer.newRow(key); - writer.addColumn(bytes("col"), bytes("val"), 1); - writer.close(); - - Map> sstables = dir.sstableLister().list(); - assertEquals(1, sstables.size()); - - Map.Entry> sstableToOpen = sstables.entrySet().iterator().next(); - final SSTableReader sstable1 = SSTableReader.open(sstableToOpen.getKey()); - - // simulate incomplete compaction - writer = new SSTableSimpleWriter(dir.getDirectoryForNewSSTables(), - cfmeta, StorageService.getPartitioner()) - { - protected SSTableWriter getWriter() - { - MetadataCollector collector = new MetadataCollector(cfmeta.comparator); - collector.addAncestor(sstable1.descriptor.generation); // add ancestor from previously written sstable - return SSTableWriter.create(createDescriptor(directory, metadata.ksName, metadata.cfName, DatabaseDescriptor.getSSTableFormat()), - 0L, - ActiveRepairService.UNREPAIRED_SSTABLE, - metadata, - DatabaseDescriptor.getPartitioner(), - collector); - } - }; - writer.newRow(key); - writer.addColumn(bytes("col"), bytes("val"), 1); - writer.close(); - - // should have 2 sstables now - sstables = dir.sstableLister().list(); - assertEquals(2, sstables.size()); - - SSTableReader sstable2 = SSTableReader.open(sstable1.descriptor); - UUID compactionTaskID = SystemKeyspace.startCompaction( - Keyspace.open(ks).getColumnFamilyStore(cf), - Collections.singleton(sstable2)); - - Map unfinishedCompaction = new HashMap<>(); - unfinishedCompaction.put(sstable1.descriptor.generation, compactionTaskID); - ColumnFamilyStore.removeUnfinishedCompactionLeftovers(cfmeta, unfinishedCompaction); - - // 2nd sstable should be removed (only 1st sstable exists in set of size 1) - sstables = dir.sstableLister().list(); - assertEquals(1, sstables.size()); - assertTrue(sstables.containsKey(sstable1.descriptor)); - - Map, Map> unfinished = SystemKeyspace.getUnfinishedCompactions(); - assertTrue(unfinished.isEmpty()); - sstable1.selfRef().release(); - sstable2.selfRef().release(); - } - - /** - * @see CASSANDRA-6086 - */ - @Test - public void testFailedToRemoveUnfinishedCompactionLeftovers() throws Throwable - { - final String ks = KEYSPACE1; - final String cf = CF_STANDARD4; // should be empty - - final CFMetaData cfmeta = Schema.instance.getCFMetaData(ks, cf); - Directories dir = new Directories(cfmeta); - ByteBuffer key = bytes("key"); - - // Write SSTable generation 3 that has ancestors 1 and 2 - final Set ancestors = Sets.newHashSet(1, 2); - SSTableSimpleWriter writer = new SSTableSimpleWriter(dir.getDirectoryForNewSSTables(), - cfmeta, StorageService.getPartitioner()) - { - protected SSTableWriter getWriter() - { - MetadataCollector collector = new MetadataCollector(cfmeta.comparator); - for (int ancestor : ancestors) - collector.addAncestor(ancestor); - String file = new Descriptor(directory, ks, cf, 3, Descriptor.Type.TEMP).filenameFor(Component.DATA); - return SSTableWriter.create(Descriptor.fromFilename(file), - 0L, - ActiveRepairService.UNREPAIRED_SSTABLE, - metadata, - StorageService.getPartitioner(), - collector); - } - }; - writer.newRow(key); - writer.addColumn(bytes("col"), bytes("val"), 1); - writer.close(); - - Map> sstables = dir.sstableLister().list(); - assert sstables.size() == 1; - - Map.Entry> sstableToOpen = sstables.entrySet().iterator().next(); - final SSTableReader sstable1 = SSTableReader.open(sstableToOpen.getKey()); - - // simulate we don't have generation in compaction_history - Map unfinishedCompactions = new HashMap<>(); - UUID compactionTaskID = UUID.randomUUID(); - for (Integer ancestor : ancestors) - unfinishedCompactions.put(ancestor, compactionTaskID); - ColumnFamilyStore.removeUnfinishedCompactionLeftovers(cfmeta, unfinishedCompactions); - - // SSTable should not be deleted - sstables = dir.sstableLister().list(); - assert sstables.size() == 1; - assert sstables.containsKey(sstable1.descriptor); - } - - @Test - public void testLoadNewSSTablesAvoidsOverwrites() throws Throwable - { - String ks = KEYSPACE1; - String cf = CF_STANDARD5; - ColumnFamilyStore cfs = Keyspace.open(ks).getColumnFamilyStore(cf); - cfs.truncateBlocking(); - SSTableDeletingTask.waitForDeletions(); - - final CFMetaData cfmeta = Schema.instance.getCFMetaData(ks, cf); - Directories dir = new Directories(cfs.metadata); - - // clear old SSTables (probably left by CFS.clearUnsafe() calls in other tests) - for (Map.Entry> entry : dir.sstableLister().list().entrySet()) - { - for (Component component : entry.getValue()) - { - FileUtils.delete(entry.getKey().filenameFor(component)); } } - - // sanity check - int existingSSTables = dir.sstableLister().list().keySet().size(); - assert existingSSTables == 0 : String.format("%d SSTables unexpectedly exist", existingSSTables); - - ByteBuffer key = bytes("key"); - - SSTableSimpleWriter writer = new SSTableSimpleWriter(dir.getDirectoryForNewSSTables(), - cfmeta, StorageService.getPartitioner()) - { - @Override - protected SSTableWriter getWriter() - { - // hack for reset generation - generation.set(0); - return super.getWriter(); - } - }; - writer.newRow(key); - writer.addColumn(bytes("col"), bytes("val"), 1); - writer.close(); - - writer = new SSTableSimpleWriter(dir.getDirectoryForNewSSTables(), - cfmeta, StorageService.getPartitioner()); - writer.newRow(key); - writer.addColumn(bytes("col"), bytes("val"), 1); - writer.close(); - - Set generations = new HashSet<>(); - for (Descriptor descriptor : dir.sstableLister().list().keySet()) - generations.add(descriptor.generation); - - // we should have two generations: [1, 2] - assertEquals(2, generations.size()); - assertTrue(generations.contains(1)); - assertTrue(generations.contains(2)); - - assertEquals(0, cfs.getSSTables().size()); - - // start the generation counter at 1 again (other tests have incremented it already) - cfs.resetFileIndexGenerator(); - - boolean incrementalBackupsEnabled = DatabaseDescriptor.isIncrementalBackupsEnabled(); - try - { - // avoid duplicate hardlinks to incremental backups - DatabaseDescriptor.setIncrementalBackupsEnabled(false); - cfs.loadNewSSTables(); - } - finally - { - DatabaseDescriptor.setIncrementalBackupsEnabled(incrementalBackupsEnabled); - } - - assertEquals(2, cfs.getSSTables().size()); - generations = new HashSet<>(); - for (Descriptor descriptor : dir.sstableLister().list().keySet()) - generations.add(descriptor.generation); - - // normally they would get renamed to generations 1 and 2, but since those filenames already exist, - // they get skipped and we end up with generations 3 and 4 - assertEquals(2, generations.size()); - assertTrue(generations.contains(3)); - assertTrue(generations.contains(4)); - } - - private ColumnFamilyStore prepareMultiRangeSlicesTest(int valueSize, boolean flush) throws Throwable - { - String keyspaceName = KEYSPACE1; - String cfName = CF_STANDARD1; - Keyspace keyspace = Keyspace.open(keyspaceName); - ColumnFamilyStore cfs = keyspace.getColumnFamilyStore(cfName); - cfs.clearUnsafe(); - - String[] letters = new String[] { "a", "b", "c", "d", "e", "f", "g", "h", "i", "j", "k", "l" }; - Cell[] cols = new Cell[12]; - for (int i = 0; i < cols.length; i++) - { - cols[i] = new BufferCell(cellname("col" + letters[i]), ByteBuffer.wrap(new byte[valueSize]), 1); - } - - for (int i = 0; i < 12; i++) - { - putColsStandard(cfs, dk(letters[i]), Arrays.copyOfRange(cols, 0, i + 1)); - } - - if (flush) - { - cfs.forceBlockingFlush(); - } - else - { - // The intent is to validate memtable code, so check we really didn't flush - assert cfs.getSSTables().isEmpty(); - } - - return cfs; - } - - private void testMultiRangeSlicesBehavior(ColumnFamilyStore cfs) - { - // in order not to change thrift interfaces at this stage we build SliceQueryFilter - // directly instead of using QueryFilter to build it for us - ColumnSlice[] startMiddleAndEndRanges = new ColumnSlice[] { - new ColumnSlice(Composites.EMPTY, cellname("colc")), - new ColumnSlice(cellname("colf"), cellname("colg")), - new ColumnSlice(cellname("colj"), Composites.EMPTY) }; - - ColumnSlice[] startMiddleAndEndRangesReversed = new ColumnSlice[] { - new ColumnSlice(Composites.EMPTY, cellname("colj")), - new ColumnSlice(cellname("colg"), cellname("colf")), - new ColumnSlice(cellname("colc"), Composites.EMPTY) }; - - ColumnSlice[] startOnlyRange = - new ColumnSlice[] { new ColumnSlice(Composites.EMPTY, cellname("colc")) }; - - ColumnSlice[] startOnlyRangeReversed = - new ColumnSlice[] { new ColumnSlice(cellname("colc"), Composites.EMPTY) }; - - ColumnSlice[] middleOnlyRanges = - new ColumnSlice[] { new ColumnSlice(cellname("colf"), cellname("colg")) }; - - ColumnSlice[] middleOnlyRangesReversed = - new ColumnSlice[] { new ColumnSlice(cellname("colg"), cellname("colf")) }; - - ColumnSlice[] endOnlyRanges = - new ColumnSlice[] { new ColumnSlice(cellname("colj"), Composites.EMPTY) }; - - ColumnSlice[] endOnlyRangesReversed = - new ColumnSlice[] { new ColumnSlice(Composites.EMPTY, cellname("colj")) }; - - SliceQueryFilter startOnlyFilter = new SliceQueryFilter(startOnlyRange, false, - Integer.MAX_VALUE); - SliceQueryFilter startOnlyFilterReversed = new SliceQueryFilter(startOnlyRangeReversed, true, - Integer.MAX_VALUE); - SliceQueryFilter startOnlyFilterWithCounting = new SliceQueryFilter(startOnlyRange, false, 1); - SliceQueryFilter startOnlyFilterReversedWithCounting = new SliceQueryFilter(startOnlyRangeReversed, - true, 1); - - SliceQueryFilter middleOnlyFilter = new SliceQueryFilter(middleOnlyRanges, - false, - Integer.MAX_VALUE); - SliceQueryFilter middleOnlyFilterReversed = new SliceQueryFilter(middleOnlyRangesReversed, true, - Integer.MAX_VALUE); - SliceQueryFilter middleOnlyFilterWithCounting = new SliceQueryFilter(middleOnlyRanges, false, 1); - SliceQueryFilter middleOnlyFilterReversedWithCounting = new SliceQueryFilter(middleOnlyRangesReversed, - true, 1); - - SliceQueryFilter endOnlyFilter = new SliceQueryFilter(endOnlyRanges, false, - Integer.MAX_VALUE); - SliceQueryFilter endOnlyReversed = new SliceQueryFilter(endOnlyRangesReversed, true, - Integer.MAX_VALUE); - SliceQueryFilter endOnlyWithCounting = new SliceQueryFilter(endOnlyRanges, false, 1); - SliceQueryFilter endOnlyWithReversedCounting = new SliceQueryFilter(endOnlyRangesReversed, - true, 1); - - SliceQueryFilter startMiddleAndEndFilter = new SliceQueryFilter(startMiddleAndEndRanges, false, - Integer.MAX_VALUE); - SliceQueryFilter startMiddleAndEndFilterReversed = new SliceQueryFilter(startMiddleAndEndRangesReversed, true, - Integer.MAX_VALUE); - SliceQueryFilter startMiddleAndEndFilterWithCounting = new SliceQueryFilter(startMiddleAndEndRanges, false, - 1); - SliceQueryFilter startMiddleAndEndFilterReversedWithCounting = new SliceQueryFilter( - startMiddleAndEndRangesReversed, true, - 1); - - findRowGetSlicesAndAssertColsFound(cfs, startOnlyFilter, "a", "cola"); - findRowGetSlicesAndAssertColsFound(cfs, startOnlyFilterReversed, "a", "cola"); - findRowGetSlicesAndAssertColsFound(cfs, startOnlyFilterWithCounting, "a", "cola"); - findRowGetSlicesAndAssertColsFound(cfs, startOnlyFilterReversedWithCounting, "a", "cola"); - - findRowGetSlicesAndAssertColsFound(cfs, middleOnlyFilter, "a", new String[] {}); - findRowGetSlicesAndAssertColsFound(cfs, middleOnlyFilterReversed, "a", new String[] {}); - findRowGetSlicesAndAssertColsFound(cfs, middleOnlyFilterWithCounting, "a", new String[] {}); - findRowGetSlicesAndAssertColsFound(cfs, middleOnlyFilterReversedWithCounting, "a", new String[] {}); - - findRowGetSlicesAndAssertColsFound(cfs, endOnlyFilter, "a", new String[] {}); - findRowGetSlicesAndAssertColsFound(cfs, endOnlyReversed, "a", new String[] {}); - findRowGetSlicesAndAssertColsFound(cfs, endOnlyWithCounting, "a", new String[] {}); - findRowGetSlicesAndAssertColsFound(cfs, endOnlyWithReversedCounting, "a", new String[] {}); - - findRowGetSlicesAndAssertColsFound(cfs, startMiddleAndEndFilter, "a", "cola"); - findRowGetSlicesAndAssertColsFound(cfs, startMiddleAndEndFilterReversed, "a", "cola"); - findRowGetSlicesAndAssertColsFound(cfs, startMiddleAndEndFilterWithCounting, "a", "cola"); - findRowGetSlicesAndAssertColsFound(cfs, startMiddleAndEndFilterReversedWithCounting, "a", "cola"); - - findRowGetSlicesAndAssertColsFound(cfs, startOnlyFilter, "c", "cola", "colb", "colc"); - findRowGetSlicesAndAssertColsFound(cfs, startOnlyFilterReversed, "c", "colc", "colb", "cola"); - findRowGetSlicesAndAssertColsFound(cfs, startOnlyFilterWithCounting, "c", "cola"); - findRowGetSlicesAndAssertColsFound(cfs, startOnlyFilterReversedWithCounting, "c", "colc"); - - findRowGetSlicesAndAssertColsFound(cfs, middleOnlyFilter, "c", new String[] {}); - findRowGetSlicesAndAssertColsFound(cfs, middleOnlyFilterReversed, "c", new String[] {}); - findRowGetSlicesAndAssertColsFound(cfs, middleOnlyFilterWithCounting, "c", new String[] {}); - findRowGetSlicesAndAssertColsFound(cfs, middleOnlyFilterReversedWithCounting, "c", new String[] {}); - - findRowGetSlicesAndAssertColsFound(cfs, endOnlyFilter, "c", new String[] {}); - findRowGetSlicesAndAssertColsFound(cfs, endOnlyReversed, "c", new String[] {}); - findRowGetSlicesAndAssertColsFound(cfs, endOnlyWithCounting, "c", new String[] {}); - findRowGetSlicesAndAssertColsFound(cfs, endOnlyWithReversedCounting, "c", new String[] {}); - - findRowGetSlicesAndAssertColsFound(cfs, startMiddleAndEndFilter, "c", "cola", "colb", "colc"); - findRowGetSlicesAndAssertColsFound(cfs, startMiddleAndEndFilterReversed, "c", "colc", "colb", "cola"); - findRowGetSlicesAndAssertColsFound(cfs, startMiddleAndEndFilterWithCounting, "c", "cola"); - findRowGetSlicesAndAssertColsFound(cfs, startMiddleAndEndFilterReversedWithCounting, "c", "colc"); - - findRowGetSlicesAndAssertColsFound(cfs, startOnlyFilter, "f", "cola", "colb", "colc"); - findRowGetSlicesAndAssertColsFound(cfs, startOnlyFilterReversed, "f", "colc", "colb", "cola"); - findRowGetSlicesAndAssertColsFound(cfs, startOnlyFilterWithCounting, "f", "cola"); - findRowGetSlicesAndAssertColsFound(cfs, startOnlyFilterReversedWithCounting, "f", "colc"); - - findRowGetSlicesAndAssertColsFound(cfs, middleOnlyFilter, "f", "colf"); - - findRowGetSlicesAndAssertColsFound(cfs, middleOnlyFilterReversed, "f", "colf"); - findRowGetSlicesAndAssertColsFound(cfs, middleOnlyFilterWithCounting, "f", "colf"); - findRowGetSlicesAndAssertColsFound(cfs, middleOnlyFilterReversedWithCounting, "f", "colf"); - - findRowGetSlicesAndAssertColsFound(cfs, endOnlyFilter, "f", new String[] {}); - findRowGetSlicesAndAssertColsFound(cfs, endOnlyReversed, "f", new String[] {}); - findRowGetSlicesAndAssertColsFound(cfs, endOnlyWithCounting, "f", new String[] {}); - findRowGetSlicesAndAssertColsFound(cfs, endOnlyWithReversedCounting, "f", new String[] {}); - - findRowGetSlicesAndAssertColsFound(cfs, startMiddleAndEndFilter, "f", "cola", "colb", "colc", "colf"); - - findRowGetSlicesAndAssertColsFound(cfs, startMiddleAndEndFilterReversed, "f", "colf", "colc", "colb", - "cola"); - findRowGetSlicesAndAssertColsFound(cfs, startMiddleAndEndFilterWithCounting, "f", "cola"); - findRowGetSlicesAndAssertColsFound(cfs, startMiddleAndEndFilterReversedWithCounting, "f", "colf"); - - findRowGetSlicesAndAssertColsFound(cfs, startOnlyFilter, "h", "cola", "colb", "colc"); - findRowGetSlicesAndAssertColsFound(cfs, startOnlyFilterReversed, "h", "colc", "colb", "cola"); - findRowGetSlicesAndAssertColsFound(cfs, startOnlyFilterWithCounting, "h", "cola"); - findRowGetSlicesAndAssertColsFound(cfs, startOnlyFilterReversedWithCounting, "h", "colc"); - - findRowGetSlicesAndAssertColsFound(cfs, middleOnlyFilter, "h", "colf", "colg"); - findRowGetSlicesAndAssertColsFound(cfs, middleOnlyFilterReversed, "h", "colg", "colf"); - findRowGetSlicesAndAssertColsFound(cfs, middleOnlyFilterWithCounting, "h", "colf"); - findRowGetSlicesAndAssertColsFound(cfs, middleOnlyFilterReversedWithCounting, "h", "colg"); - - findRowGetSlicesAndAssertColsFound(cfs, endOnlyFilter, "h", new String[] {}); - findRowGetSlicesAndAssertColsFound(cfs, endOnlyReversed, "h", new String[] {}); - findRowGetSlicesAndAssertColsFound(cfs, endOnlyWithCounting, "h", new String[] {}); - findRowGetSlicesAndAssertColsFound(cfs, endOnlyWithReversedCounting, "h", new String[] {}); - - findRowGetSlicesAndAssertColsFound(cfs, startMiddleAndEndFilter, "h", "cola", "colb", "colc", "colf", - "colg"); - findRowGetSlicesAndAssertColsFound(cfs, startMiddleAndEndFilterReversed, "h", "colg", "colf", "colc", "colb", - "cola"); - findRowGetSlicesAndAssertColsFound(cfs, startMiddleAndEndFilterWithCounting, "h", "cola"); - findRowGetSlicesAndAssertColsFound(cfs, startMiddleAndEndFilterReversedWithCounting, "h", "colg"); - - findRowGetSlicesAndAssertColsFound(cfs, startOnlyFilter, "j", "cola", "colb", "colc"); - findRowGetSlicesAndAssertColsFound(cfs, startOnlyFilterReversed, "j", "colc", "colb", "cola"); - findRowGetSlicesAndAssertColsFound(cfs, startOnlyFilterWithCounting, "j", "cola"); - findRowGetSlicesAndAssertColsFound(cfs, startOnlyFilterReversedWithCounting, "j", "colc"); - - findRowGetSlicesAndAssertColsFound(cfs, middleOnlyFilter, "j", "colf", "colg"); - findRowGetSlicesAndAssertColsFound(cfs, middleOnlyFilterReversed, "j", "colg", "colf"); - findRowGetSlicesAndAssertColsFound(cfs, middleOnlyFilterWithCounting, "j", "colf"); - findRowGetSlicesAndAssertColsFound(cfs, middleOnlyFilterReversedWithCounting, "j", "colg"); - - findRowGetSlicesAndAssertColsFound(cfs, endOnlyFilter, "j", "colj"); - findRowGetSlicesAndAssertColsFound(cfs, endOnlyReversed, "j", "colj"); - findRowGetSlicesAndAssertColsFound(cfs, endOnlyWithCounting, "j", "colj"); - findRowGetSlicesAndAssertColsFound(cfs, endOnlyWithReversedCounting, "j", "colj"); - - findRowGetSlicesAndAssertColsFound(cfs, startMiddleAndEndFilter, "j", "cola", "colb", "colc", "colf", "colg", - "colj"); - findRowGetSlicesAndAssertColsFound(cfs, startMiddleAndEndFilterReversed, "j", "colj", "colg", "colf", "colc", - "colb", "cola"); - findRowGetSlicesAndAssertColsFound(cfs, startMiddleAndEndFilterWithCounting, "j", "cola"); - findRowGetSlicesAndAssertColsFound(cfs, startMiddleAndEndFilterReversedWithCounting, "j", "colj"); - - findRowGetSlicesAndAssertColsFound(cfs, startOnlyFilter, "l", "cola", "colb", "colc"); - findRowGetSlicesAndAssertColsFound(cfs, startOnlyFilterReversed, "l", "colc", "colb", "cola"); - findRowGetSlicesAndAssertColsFound(cfs, startOnlyFilterWithCounting, "l", "cola"); - findRowGetSlicesAndAssertColsFound(cfs, startOnlyFilterReversedWithCounting, "l", "colc"); - - findRowGetSlicesAndAssertColsFound(cfs, middleOnlyFilter, "l", "colf", "colg"); - findRowGetSlicesAndAssertColsFound(cfs, middleOnlyFilterReversed, "l", "colg", "colf"); - findRowGetSlicesAndAssertColsFound(cfs, middleOnlyFilterWithCounting, "l", "colf"); - findRowGetSlicesAndAssertColsFound(cfs, middleOnlyFilterReversedWithCounting, "l", "colg"); - - findRowGetSlicesAndAssertColsFound(cfs, endOnlyFilter, "l", "colj", "colk", "coll"); - findRowGetSlicesAndAssertColsFound(cfs, endOnlyReversed, "l", "coll", "colk", "colj"); - findRowGetSlicesAndAssertColsFound(cfs, endOnlyWithCounting, "l", "colj"); - findRowGetSlicesAndAssertColsFound(cfs, endOnlyWithReversedCounting, "l", "coll"); - - findRowGetSlicesAndAssertColsFound(cfs, startMiddleAndEndFilter, "l", "cola", "colb", "colc", "colf", "colg", - "colj", "colk", "coll"); - findRowGetSlicesAndAssertColsFound(cfs, startMiddleAndEndFilterReversed, "l", "coll", "colk", "colj", "colg", - "colf", "colc", "colb", "cola"); - findRowGetSlicesAndAssertColsFound(cfs, startMiddleAndEndFilterWithCounting, "l", "cola"); - findRowGetSlicesAndAssertColsFound(cfs, startMiddleAndEndFilterReversedWithCounting, "l", "coll"); - } - - private void findRowGetSlicesAndAssertColsFound(ColumnFamilyStore cfs, SliceQueryFilter filter, String rowKey, - String... colNames) - { - List rows = cfs.getRangeSlice(new Bounds(rp(rowKey), rp(rowKey)), - null, - filter, - Integer.MAX_VALUE, - System.currentTimeMillis(), - false, - false); - assertSame("unexpected number of rows ", 1, rows.size()); - Row row = rows.get(0); - Collection cols = !filter.isReversed() ? row.cf.getSortedColumns() : row.cf.getReverseSortedColumns(); - // printRow(cfs, new String(row.key.key.array()), cols); - String[] returnedColsNames = Iterables.toArray(Iterables.transform(cols, new Function() - { - public String apply(Cell arg0) - { - return Util.string(arg0.name().toByteBuffer()); - } - }), String.class); - - assertTrue( - "Columns did not match. Expected: " + Arrays.toString(colNames) + " but got:" - + Arrays.toString(returnedColsNames), Arrays.equals(colNames, returnedColsNames)); - int i = 0; - for (Cell col : cols) - { - assertEquals(colNames[i++], Util.string(col.name().toByteBuffer())); - } - } - - private void printRow(ColumnFamilyStore cfs, String rowKey, Collection cols) - { - DecoratedKey ROW = Util.dk(rowKey); - System.err.println("Original:"); - ColumnFamily cf = cfs.getColumnFamily(QueryFilter.getIdentityFilter(ROW, CF_STANDARD1, System.currentTimeMillis())); - System.err.println("Row key: " + rowKey + " Cols: " - + Iterables.transform(cf.getSortedColumns(), new Function() - { - public String apply(Cell arg0) - { - return Util.string(arg0.name().toByteBuffer()); - } - })); - System.err.println("Filtered:"); - Iterable transformed = Iterables.transform(cols, new Function() - { - public String apply(Cell arg0) - { - return Util.string(arg0.name().toByteBuffer()); - } - }); - System.err.println("Row key: " + rowKey + " Cols: " + transformed); - } - - @Test - public void testRebuildSecondaryIndex() throws IOException - { - CellName indexedCellName = cellname("indexed"); - Mutation rm; - - rm = new Mutation(KEYSPACE4, ByteBufferUtil.bytes("k1")); - rm.add("Indexed1", indexedCellName, ByteBufferUtil.bytes("foo"), 1); - - rm.apply(); - assertTrue(Arrays.equals("k1".getBytes(), PerRowSecondaryIndexTest.TestIndex.LAST_INDEXED_KEY.array())); - - Keyspace.open("PerRowSecondaryIndex").getColumnFamilyStore("Indexed1").forceBlockingFlush(); - - PerRowSecondaryIndexTest.TestIndex.reset(); - - ColumnFamilyStore.rebuildSecondaryIndex("PerRowSecondaryIndex", "Indexed1", PerRowSecondaryIndexTest.TestIndex.class.getSimpleName()); - assertTrue(Arrays.equals("k1".getBytes(), PerRowSecondaryIndexTest.TestIndex.LAST_INDEXED_KEY.array())); - - PerRowSecondaryIndexTest.TestIndex.reset(); - PerRowSecondaryIndexTest.TestIndex.ACTIVE = false; - ColumnFamilyStore.rebuildSecondaryIndex("PerRowSecondaryIndex", "Indexed1", PerRowSecondaryIndexTest.TestIndex.class.getSimpleName()); - assertNull(PerRowSecondaryIndexTest.TestIndex.LAST_INDEXED_KEY); - - PerRowSecondaryIndexTest.TestIndex.reset(); + assertEquals(count, found); } } diff --git a/test/unit/org/apache/cassandra/db/ColumnFamilyTest.java b/test/unit/org/apache/cassandra/db/ColumnFamilyTest.java deleted file mode 100644 index 72ddd40fc5..0000000000 --- a/test/unit/org/apache/cassandra/db/ColumnFamilyTest.java +++ /dev/null @@ -1,277 +0,0 @@ -/* -* Licensed to the Apache Software Foundation (ASF) under one -* or more contributor license agreements. See the NOTICE file -* distributed with this work for additional information -* regarding copyright ownership. The ASF licenses this file -* to you under the Apache License, Version 2.0 (the -* "License"); you may not use this file except in compliance -* with the License. You may obtain a copy of the License at -* -* http://www.apache.org/licenses/LICENSE-2.0 -* -* Unless required by applicable law or agreed to in writing, -* software distributed under the License is distributed on an -* "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -* KIND, either express or implied. See the License for the -* specific language governing permissions and limitations -* under the License. -*/ -package org.apache.cassandra.db; - -import java.io.ByteArrayInputStream; -import java.io.DataInputStream; -import java.io.IOException; -import java.nio.ByteBuffer; -import java.util.TreeMap; - -import com.google.common.collect.Iterables; -import org.apache.cassandra.config.CFMetaData; -import org.junit.BeforeClass; -import org.junit.Test; - -import org.apache.cassandra.SchemaLoader; -import org.apache.cassandra.config.KSMetaData; -import org.apache.cassandra.exceptions.ConfigurationException; -import org.apache.cassandra.db.composites.CellName; -import org.apache.cassandra.db.context.CounterContext; -import org.apache.cassandra.db.marshal.BytesType; -import org.apache.cassandra.db.marshal.CounterColumnType; -import org.apache.cassandra.io.sstable.ColumnStats; -import org.apache.cassandra.io.util.DataOutputBuffer; -import org.apache.cassandra.locator.SimpleStrategy; -import org.apache.cassandra.net.MessagingService; -import org.apache.cassandra.utils.ByteBufferUtil; -import org.apache.cassandra.utils.CounterId; -import org.apache.cassandra.utils.FBUtilities; - -import static junit.framework.Assert.assertTrue; - -import static org.apache.cassandra.Util.column; -import static org.apache.cassandra.Util.cellname; -import static org.apache.cassandra.Util.tombstone; -import static org.junit.Assert.assertEquals; -import static org.junit.Assert.assertFalse; - -public class ColumnFamilyTest -{ - static int version = MessagingService.current_version; - private static final String KEYSPACE1 = "Keyspace1"; - private static final String CF_STANDARD1 = "Standard1"; - private static final String CF_COUNTER1 = "Counter1"; - - @BeforeClass - public static void defineSchema() throws ConfigurationException - { - SchemaLoader.prepareServer(); - SchemaLoader.createKeyspace(KEYSPACE1, - SimpleStrategy.class, - KSMetaData.optsWithRF(1), - SchemaLoader.standardCFMD(KEYSPACE1, CF_STANDARD1), - SchemaLoader.standardCFMD(KEYSPACE1, CF_COUNTER1) - .defaultValidator(CounterColumnType.instance)); - } - - // TODO test SuperColumns more - - @Test - public void testSingleColumn() throws IOException - { - ColumnFamily cf; - - cf = ArrayBackedSortedColumns.factory.create(KEYSPACE1, CF_STANDARD1); - cf.addColumn(column("C", "v", 1)); - DataOutputBuffer bufOut = new DataOutputBuffer(); - ColumnFamily.serializer.serialize(cf, bufOut, version); - - ByteArrayInputStream bufIn = new ByteArrayInputStream(bufOut.getData(), 0, bufOut.getLength()); - cf = ColumnFamily.serializer.deserialize(new DataInputStream(bufIn), version); - assert cf != null; - assert cf.metadata().cfName.equals(CF_STANDARD1); - assert cf.getSortedColumns().size() == 1; - } - - @Test - public void testManyColumns() throws IOException - { - ColumnFamily cf; - - TreeMap map = new TreeMap<>(); - for (int i = 100; i < 1000; ++i) - { - map.put(Integer.toString(i), "Avinash Lakshman is a good man: " + i); - } - - // write - cf = ArrayBackedSortedColumns.factory.create(KEYSPACE1, CF_STANDARD1); - DataOutputBuffer bufOut = new DataOutputBuffer(); - for (String cName : map.navigableKeySet()) - { - cf.addColumn(column(cName, map.get(cName), 314)); - } - ColumnFamily.serializer.serialize(cf, bufOut, version); - - // verify - ByteArrayInputStream bufIn = new ByteArrayInputStream(bufOut.getData(), 0, bufOut.getLength()); - cf = ColumnFamily.serializer.deserialize(new DataInputStream(bufIn), version); - for (String cName : map.navigableKeySet()) - { - ByteBuffer val = cf.getColumn(cellname(cName)).value(); - assert new String(val.array(),val.position(),val.remaining()).equals(map.get(cName)); - } - assert Iterables.size(cf.getColumnNames()) == map.size(); - } - - @Test - public void testGetColumnCount() - { - ColumnFamily cf = ArrayBackedSortedColumns.factory.create(KEYSPACE1, CF_STANDARD1); - - cf.addColumn(column("col1", "", 1)); - cf.addColumn(column("col2", "", 2)); - cf.addColumn(column("col1", "", 3)); - - assert 2 == cf.getColumnCount(); - assert 2 == cf.getSortedColumns().size(); - } - - @Test - public void testDigest() - { - ColumnFamily cf = ArrayBackedSortedColumns.factory.create(KEYSPACE1, CF_STANDARD1); - ColumnFamily cf2 = ArrayBackedSortedColumns.factory.create(KEYSPACE1, CF_STANDARD1); - - ByteBuffer digest = ColumnFamily.digest(cf); - - cf.addColumn(column("col1", "", 1)); - cf2.addColumn(column("col1", "", 1)); - - assert !digest.equals(ColumnFamily.digest(cf)); - - digest = ColumnFamily.digest(cf); - assert digest.equals(ColumnFamily.digest(cf2)); - - cf.addColumn(column("col2", "", 2)); - assert !digest.equals(ColumnFamily.digest(cf)); - - digest = ColumnFamily.digest(cf); - cf.addColumn(column("col1", "", 3)); - assert !digest.equals(ColumnFamily.digest(cf)); - - digest = ColumnFamily.digest(cf); - cf.delete(new DeletionTime(4, 4)); - assert !digest.equals(ColumnFamily.digest(cf)); - - digest = ColumnFamily.digest(cf); - cf.delete(tombstone("col1", "col11", 5, 5)); - assert !digest.equals(ColumnFamily.digest(cf)); - - digest = ColumnFamily.digest(cf); - assert digest.equals(ColumnFamily.digest(cf)); - - cf.delete(tombstone("col2", "col21", 5, 5)); - assert !digest.equals(ColumnFamily.digest(cf)); - - digest = ColumnFamily.digest(cf); - cf.delete(tombstone("col1", "col11", 5, 5)); // this does not change RangeTombstoneLList - assert digest.equals(ColumnFamily.digest(cf)); - } - - @Test - public void testTimestamp() - { - ColumnFamily cf = ArrayBackedSortedColumns.factory.create(KEYSPACE1, CF_STANDARD1); - - cf.addColumn(column("col1", "val1", 2)); - cf.addColumn(column("col1", "val2", 2)); // same timestamp, new value - cf.addColumn(column("col1", "val3", 1)); // older timestamp -- should be ignored - - assert ByteBufferUtil.bytes("val2").equals(cf.getColumn(cellname("col1")).value()); - } - - @Test - public void testMergeAndAdd() - { - ColumnFamily cf_new = ArrayBackedSortedColumns.factory.create(KEYSPACE1, CF_STANDARD1); - ColumnFamily cf_old = ArrayBackedSortedColumns.factory.create(KEYSPACE1, CF_STANDARD1); - ColumnFamily cf_result = ArrayBackedSortedColumns.factory.create(KEYSPACE1, CF_STANDARD1); - ByteBuffer val = ByteBufferUtil.bytes("sample value"); - ByteBuffer val2 = ByteBufferUtil.bytes("x value "); - - cf_new.addColumn(cellname("col1"), val, 3); - cf_new.addColumn(cellname("col2"), val, 4); - - cf_old.addColumn(cellname("col2"), val2, 1); - cf_old.addColumn(cellname("col3"), val2, 2); - - cf_result.addAll(cf_new); - cf_result.addAll(cf_old); - - assert 3 == cf_result.getColumnCount() : "Count is " + cf_new.getColumnCount(); - //addcolumns will only add if timestamp >= old timestamp - assert val.equals(cf_result.getColumn(cellname("col2")).value()); - - // check that tombstone wins timestamp ties - cf_result.addTombstone(cellname("col1"), 0, 3); - assertFalse(cf_result.getColumn(cellname("col1")).isLive()); - cf_result.addColumn(cellname("col1"), val2, 3); - assertFalse(cf_result.getColumn(cellname("col1")).isLive()); - - // check that column value wins timestamp ties in absence of tombstone - cf_result.addColumn(cellname("col3"), val, 2); - assert cf_result.getColumn(cellname("col3")).value().equals(val2); - cf_result.addColumn(cellname("col3"), ByteBufferUtil.bytes("z"), 2); - assert cf_result.getColumn(cellname("col3")).value().equals(ByteBufferUtil.bytes("z")); - } - - @Test - public void testColumnStatsRecordsRowDeletesCorrectly() - { - long timestamp = System.currentTimeMillis(); - int localDeletionTime = (int) (System.currentTimeMillis() / 1000); - - ColumnFamily cf = ArrayBackedSortedColumns.factory.create(KEYSPACE1, CF_STANDARD1); - cf.delete(new DeletionInfo(timestamp, localDeletionTime)); - ColumnStats stats = cf.getColumnStats(); - assertEquals(timestamp, stats.maxTimestamp); - - cf.delete(new RangeTombstone(cellname("col2"), cellname("col21"), timestamp, localDeletionTime)); - - stats = cf.getColumnStats(); - assertEquals(ByteBufferUtil.bytes("col2"), stats.minColumnNames.get(0)); - assertEquals(ByteBufferUtil.bytes("col21"), stats.maxColumnNames.get(0)); - - cf.delete(new RangeTombstone(cellname("col6"), cellname("col61"), timestamp, localDeletionTime)); - stats = cf.getColumnStats(); - - assertEquals(ByteBufferUtil.bytes("col2"), stats.minColumnNames.get(0)); - assertEquals(ByteBufferUtil.bytes("col61"), stats.maxColumnNames.get(0)); - } - - @Test - public void testCounterDeletion() - { - long timestamp = FBUtilities.timestampMicros(); - CellName name = cellname("counter1"); - - BufferCounterCell counter = new BufferCounterCell(name, - CounterContext.instance().createGlobal(CounterId.fromInt(1), 1, 1), - timestamp); - BufferDeletedCell tombstone = new BufferDeletedCell(name, (int) (System.currentTimeMillis() / 1000), 0L); - - // check that the tombstone won the reconcile despite the counter cell having a higher timestamp - assertTrue(counter.reconcile(tombstone) == tombstone); - - // check that a range tombstone overrides the counter cell, even with a lower timestamp than the counter - ColumnFamily cf0 = ArrayBackedSortedColumns.factory.create(KEYSPACE1, CF_COUNTER1); - cf0.addColumn(counter); - cf0.delete(new RangeTombstone(cellname("counter0"), cellname("counter2"), 0L, (int) (System.currentTimeMillis() / 1000))); - assertTrue(cf0.deletionInfo().isDeleted(counter)); - assertTrue(cf0.deletionInfo().inOrderTester(false).isDeleted(counter)); - - // check that a top-level deletion info overrides the counter cell, even with a lower timestamp than the counter - ColumnFamily cf1 = ArrayBackedSortedColumns.factory.create(KEYSPACE1, CF_COUNTER1); - cf1.addColumn(counter); - cf1.delete(new DeletionInfo(0L, (int) (System.currentTimeMillis() / 1000))); - assertTrue(cf1.deletionInfo().isDeleted(counter)); - } -} diff --git a/test/unit/org/apache/cassandra/db/CommitLogTest.java b/test/unit/org/apache/cassandra/db/CommitLogTest.java index 1ab678f939..70dbed032c 100644 --- a/test/unit/org/apache/cassandra/db/CommitLogTest.java +++ b/test/unit/org/apache/cassandra/db/CommitLogTest.java @@ -19,7 +19,10 @@ package org.apache.cassandra.db; +import static junit.framework.Assert.assertFalse; +import static junit.framework.Assert.assertTrue; import static org.apache.cassandra.utils.ByteBufferUtil.bytes; +import static org.junit.Assert.assertEquals; import java.io.ByteArrayOutputStream; import java.io.DataOutputStream; @@ -36,6 +39,13 @@ import java.util.zip.Checksum; import com.google.common.collect.ImmutableMap; +import org.apache.cassandra.db.rows.Row; +import org.apache.cassandra.db.marshal.AsciiType; +import org.apache.cassandra.db.marshal.BytesType; +import org.apache.cassandra.db.partitions.PartitionIterator; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + import org.junit.Assert; import org.junit.BeforeClass; import org.junit.Test; @@ -49,23 +59,21 @@ import org.apache.cassandra.db.commitlog.CommitLogDescriptor; import org.apache.cassandra.db.commitlog.CommitLogSegment; import org.apache.cassandra.db.commitlog.ReplayPosition; import org.apache.cassandra.db.compaction.CompactionManager; -import org.apache.cassandra.db.composites.CellName; -import org.apache.cassandra.db.composites.CellNameType; -import org.apache.cassandra.db.filter.NamesQueryFilter; import org.apache.cassandra.exceptions.ConfigurationException; import org.apache.cassandra.io.util.ByteBufferDataInput; import org.apache.cassandra.io.util.FileDataInput; import org.apache.cassandra.locator.SimpleStrategy; import org.apache.cassandra.net.MessagingService; import org.apache.cassandra.utils.ByteBufferUtil; -import org.apache.cassandra.utils.FBUtilities; public class CommitLogTest { + private static final Logger logger = LoggerFactory.getLogger(CommitLogTest.class); + private static final String KEYSPACE1 = "CommitLogTest"; private static final String KEYSPACE2 = "CommitLogTestNonDurable"; - private static final String CF1 = "Standard1"; - private static final String CF2 = "Standard2"; + private static final String STANDARD1 = "Standard1"; + private static final String STANDARD2 = "Standard2"; @BeforeClass public static void defineSchema() throws ConfigurationException @@ -74,15 +82,15 @@ public class CommitLogTest SchemaLoader.createKeyspace(KEYSPACE1, SimpleStrategy.class, KSMetaData.optsWithRF(1), - SchemaLoader.standardCFMD(KEYSPACE1, CF1), - SchemaLoader.standardCFMD(KEYSPACE1, CF2)); + SchemaLoader.standardCFMD(KEYSPACE1, STANDARD1, 0, AsciiType.instance, BytesType.instance), + SchemaLoader.standardCFMD(KEYSPACE1, STANDARD2, 0, AsciiType.instance, BytesType.instance)); SchemaLoader.createKeyspace(KEYSPACE2, false, true, SimpleStrategy.class, KSMetaData.optsWithRF(1), - SchemaLoader.standardCFMD(KEYSPACE1, CF1), - SchemaLoader.standardCFMD(KEYSPACE1, CF2)); + SchemaLoader.standardCFMD(KEYSPACE1, STANDARD1, 0, AsciiType.instance, BytesType.instance), + SchemaLoader.standardCFMD(KEYSPACE1, STANDARD2, 0, AsciiType.instance, BytesType.instance)); System.setProperty("cassandra.commitlog.stop_on_errors", "true"); CompactionManager.instance.disableAutoCompaction(); } @@ -146,25 +154,32 @@ public class CommitLogTest public void testDontDeleteIfDirty() throws Exception { CommitLog.instance.resetUnsafe(true); + ColumnFamilyStore cfs1 = Keyspace.open(KEYSPACE1).getColumnFamilyStore(STANDARD1); + ColumnFamilyStore cfs2 = Keyspace.open(KEYSPACE1).getColumnFamilyStore(STANDARD2); + // Roughly 32 MB mutation - Mutation rm = new Mutation(KEYSPACE1, bytes("k")); - rm.add(CF1, Util.cellname("c1"), ByteBuffer.allocate(DatabaseDescriptor.getCommitLogSegmentSize()/4), 0); + Mutation m = new RowUpdateBuilder(cfs1.metadata, 0, "k") + .clustering("bytes") + .add("val", ByteBuffer.allocate(DatabaseDescriptor.getCommitLogSegmentSize()/4)) + .build(); // Adding it 5 times - CommitLog.instance.add(rm); - CommitLog.instance.add(rm); - CommitLog.instance.add(rm); - CommitLog.instance.add(rm); - CommitLog.instance.add(rm); + CommitLog.instance.add(m); + CommitLog.instance.add(m); + CommitLog.instance.add(m); + CommitLog.instance.add(m); + CommitLog.instance.add(m); // Adding new mutation on another CF - Mutation rm2 = new Mutation(KEYSPACE1, bytes("k")); - rm2.add(CF2, Util.cellname("c1"), ByteBuffer.allocate(4), 0); - CommitLog.instance.add(rm2); + Mutation m2 = new RowUpdateBuilder(cfs2.metadata, 0, "k") + .clustering("bytes") + .add("val", ByteBuffer.allocate(4)) + .build(); + CommitLog.instance.add(m2); assert CommitLog.instance.activeSegments() == 2 : "Expecting 2 segments, got " + CommitLog.instance.activeSegments(); - UUID cfid2 = rm2.getColumnFamilyIds().iterator().next(); + UUID cfid2 = m2.getColumnFamilyIds().iterator().next(); CommitLog.instance.discardCompletedSegments(cfid2, CommitLog.instance.getContext()); // Assert we still have both our segment @@ -176,9 +191,14 @@ public class CommitLogTest { DatabaseDescriptor.getCommitLogSegmentSize(); CommitLog.instance.resetUnsafe(true); + ColumnFamilyStore cfs1 = Keyspace.open(KEYSPACE1).getColumnFamilyStore(STANDARD1); + ColumnFamilyStore cfs2 = Keyspace.open(KEYSPACE1).getColumnFamilyStore(STANDARD2); + // Roughly 32 MB mutation - Mutation rm = new Mutation(KEYSPACE1, bytes("k")); - rm.add(CF1, Util.cellname("c1"), ByteBuffer.allocate((DatabaseDescriptor.getCommitLogSegmentSize()/4) - 1), 0); + Mutation rm = new RowUpdateBuilder(cfs1.metadata, 0, "k") + .clustering("bytes") + .add("val", ByteBuffer.allocate((DatabaseDescriptor.getCommitLogSegmentSize()/4) - 1)) + .build(); // Adding it twice (won't change segment) CommitLog.instance.add(rm); @@ -194,8 +214,10 @@ public class CommitLogTest assert CommitLog.instance.activeSegments() == 1 : "Expecting 1 segment, got " + CommitLog.instance.activeSegments(); // Adding new mutation on another CF, large enough (including CL entry overhead) that a new segment is created - Mutation rm2 = new Mutation(KEYSPACE1, bytes("k")); - rm2.add(CF2, Util.cellname("c1"), ByteBuffer.allocate((DatabaseDescriptor.getCommitLogSegmentSize()/2) - 200), 0); + Mutation rm2 = new RowUpdateBuilder(cfs2.metadata, 0, "k") + .clustering("bytes") + .add("val", ByteBuffer.allocate((DatabaseDescriptor.getCommitLogSegmentSize()/2) - 200)) + .build(); CommitLog.instance.add(rm2); // also forces a new segment, since each entry-with-overhead is just under half the CL size CommitLog.instance.add(rm2); @@ -214,19 +236,24 @@ public class CommitLogTest assert CommitLog.instance.activeSegments() == 1 : "Expecting 1 segment, got " + CommitLog.instance.activeSegments(); } - private static int getMaxRecordDataSize(String keyspace, ByteBuffer key, String table, CellName column) + private static int getMaxRecordDataSize(String keyspace, ByteBuffer key, String cfName, String colName) { - Mutation rm = new Mutation(KEYSPACE1, bytes("k")); - rm.add("Standard1", Util.cellname("c1"), ByteBuffer.allocate(0), 0); + ColumnFamilyStore cfs = Keyspace.open(keyspace).getColumnFamilyStore(cfName); + // We don't want to allocate a size of 0 as this is optimize under the hood and our computation would + // break testEqualRecordLimit + int allocSize = 1; + Mutation rm = new RowUpdateBuilder(cfs.metadata, 0, key) + .clustering(colName) + .add("val", ByteBuffer.allocate(allocSize)).build(); int max = (DatabaseDescriptor.getCommitLogSegmentSize() / 2); max -= CommitLogSegment.ENTRY_OVERHEAD_SIZE; // log entry overhead - return max - (int) Mutation.serializer.serializedSize(rm, MessagingService.current_version); + return max - (int) Mutation.serializer.serializedSize(rm, MessagingService.current_version) + allocSize; } private static int getMaxRecordDataSize() { - return getMaxRecordDataSize(KEYSPACE1, bytes("k"), CF1, Util.cellname("c1")); + return getMaxRecordDataSize(KEYSPACE1, bytes("k"), STANDARD1, "bytes"); } // CASSANDRA-3615 @@ -234,9 +261,11 @@ public class CommitLogTest public void testEqualRecordLimit() throws Exception { CommitLog.instance.resetUnsafe(true); - - Mutation rm = new Mutation(KEYSPACE1, bytes("k")); - rm.add(CF1, Util.cellname("c1"), ByteBuffer.allocate(getMaxRecordDataSize()), 0); + ColumnFamilyStore cfs = Keyspace.open(KEYSPACE1).getColumnFamilyStore(STANDARD1); + Mutation rm = new RowUpdateBuilder(cfs.metadata, 0, "k") + .clustering("bytes") + .add("val", ByteBuffer.allocate(getMaxRecordDataSize())) + .build(); CommitLog.instance.add(rm); } @@ -244,10 +273,13 @@ public class CommitLogTest public void testExceedRecordLimit() throws Exception { CommitLog.instance.resetUnsafe(true); + ColumnFamilyStore cfs = Keyspace.open(KEYSPACE1).getColumnFamilyStore(STANDARD1); try { - Mutation rm = new Mutation(KEYSPACE1, bytes("k")); - rm.add(CF1, Util.cellname("c1"), ByteBuffer.allocate(1 + getMaxRecordDataSize()), 0); + Mutation rm = new RowUpdateBuilder(cfs.metadata, 0, "k") + .clustering("bytes") + .add("val", ByteBuffer.allocate(1 + getMaxRecordDataSize())) + .build(); CommitLog.instance.add(rm); throw new AssertionError("mutation larger than limit was accepted"); } @@ -257,6 +289,133 @@ public class CommitLogTest } } + @Test + public void testVersions() + { + Assert.assertTrue(CommitLogDescriptor.isValid("CommitLog-1340512736956320000.log")); + Assert.assertTrue(CommitLogDescriptor.isValid("CommitLog-2-1340512736956320000.log")); + Assert.assertFalse(CommitLogDescriptor.isValid("CommitLog--1340512736956320000.log")); + Assert.assertFalse(CommitLogDescriptor.isValid("CommitLog--2-1340512736956320000.log")); + Assert.assertFalse(CommitLogDescriptor.isValid("CommitLog-2-1340512736956320000-123.log")); + + assertEquals(1340512736956320000L, CommitLogDescriptor.fromFileName("CommitLog-2-1340512736956320000.log").id); + + assertEquals(MessagingService.current_version, new CommitLogDescriptor(1340512736956320000L, null).getMessagingVersion()); + String newCLName = "CommitLog-" + CommitLogDescriptor.current_version + "-1340512736956320000.log"; + assertEquals(MessagingService.current_version, CommitLogDescriptor.fromFileName(newCLName).getMessagingVersion()); + } + + @Test + public void testTruncateWithoutSnapshot() throws IOException + { + boolean originalState = DatabaseDescriptor.isAutoSnapshot(); + try + { + CommitLog.instance.resetUnsafe(true); + boolean prev = DatabaseDescriptor.isAutoSnapshot(); + DatabaseDescriptor.setAutoSnapshot(false); + ColumnFamilyStore cfs1 = Keyspace.open(KEYSPACE1).getColumnFamilyStore(STANDARD1); + ColumnFamilyStore cfs2 = Keyspace.open(KEYSPACE1).getColumnFamilyStore(STANDARD2); + + new RowUpdateBuilder(cfs1.metadata, 0, "k").clustering("bytes").add("val", ByteBuffer.allocate(100)).build().applyUnsafe(); + cfs1.truncateBlocking(); + DatabaseDescriptor.setAutoSnapshot(prev); + Mutation m2 = new RowUpdateBuilder(cfs2.metadata, 0, "k") + .clustering("bytes") + .add("val", ByteBuffer.allocate(DatabaseDescriptor.getCommitLogSegmentSize() / 4)) + .build(); + + for (int i = 0 ; i < 5 ; i++) + CommitLog.instance.add(m2); + + assertEquals(2, CommitLog.instance.activeSegments()); + ReplayPosition position = CommitLog.instance.getContext(); + for (Keyspace ks : Keyspace.system()) + for (ColumnFamilyStore syscfs : ks.getColumnFamilyStores()) + CommitLog.instance.discardCompletedSegments(syscfs.metadata.cfId, position); + CommitLog.instance.discardCompletedSegments(cfs2.metadata.cfId, position); + assertEquals(1, CommitLog.instance.activeSegments()); + } + finally + { + DatabaseDescriptor.setAutoSnapshot(originalState); + } + } + + @Test + public void testTruncateWithoutSnapshotNonDurable() throws IOException + { + CommitLog.instance.resetUnsafe(true); + boolean originalState = DatabaseDescriptor.getAutoSnapshot(); + try + { + DatabaseDescriptor.setAutoSnapshot(false); + Keyspace notDurableKs = Keyspace.open(KEYSPACE2); + Assert.assertFalse(notDurableKs.metadata.durableWrites); + + ColumnFamilyStore cfs = notDurableKs.getColumnFamilyStore("Standard1"); + new RowUpdateBuilder(cfs.metadata, 0, "key1") + .clustering("bytes").add("val", ByteBufferUtil.bytes("abcd")) + .build() + .applyUnsafe(); + + assertTrue(Util.getOnlyRow(Util.cmd(cfs).columns("val").build()) + .iterator().next().value().equals(ByteBufferUtil.bytes("abcd"))); + + cfs.truncateBlocking(); + + Util.assertEmpty(Util.cmd(cfs).columns("val").build()); + } + finally + { + DatabaseDescriptor.setAutoSnapshot(originalState); + } + } + + private void testDescriptorPersistence(CommitLogDescriptor desc) throws IOException + { + ByteBuffer buf = ByteBuffer.allocate(1024); + CommitLogDescriptor.writeHeader(buf, desc); + long length = buf.position(); + // Put some extra data in the stream. + buf.putDouble(0.1); + buf.flip(); + FileDataInput input = new ByteBufferDataInput(buf, "input", 0, 0); + CommitLogDescriptor read = CommitLogDescriptor.readHeader(input); + Assert.assertEquals("Descriptor length", length, input.getFilePointer()); + Assert.assertEquals("Descriptors", desc, read); + } + + @Test + public void testDescriptorPersistence() throws IOException + { + testDescriptorPersistence(new CommitLogDescriptor(11, null)); + testDescriptorPersistence(new CommitLogDescriptor(CommitLogDescriptor.VERSION_21, 13, null)); + testDescriptorPersistence(new CommitLogDescriptor(CommitLogDescriptor.VERSION_30, 15, null)); + testDescriptorPersistence(new CommitLogDescriptor(CommitLogDescriptor.VERSION_30, 17, new ParameterizedClass("LZ4Compressor", null))); + testDescriptorPersistence(new CommitLogDescriptor(CommitLogDescriptor.VERSION_30, 19, + new ParameterizedClass("StubbyCompressor", ImmutableMap.of("parameter1", "value1", "flag2", "55", "argument3", "null")))); + } + + @Test + public void testDescriptorInvalidParametersSize() throws IOException + { + Map params = new HashMap<>(); + for (int i=0; i<65535; ++i) + params.put("key"+i, Integer.toString(i, 16)); + try { + CommitLogDescriptor desc = new CommitLogDescriptor(CommitLogDescriptor.VERSION_30, + 21, + new ParameterizedClass("LZ4Compressor", params)); + ByteBuffer buf = ByteBuffer.allocate(1024000); + CommitLogDescriptor.writeHeader(buf, desc); + Assert.fail("Parameter object too long should fail on writing descriptor."); + } catch (ConfigurationException e) + { + // correct path + } + } + protected void testRecoveryWithBadSizeArgument(int size, int dataSize) throws Exception { Checksum checksum = new CRC32(); @@ -290,124 +449,8 @@ public class CommitLogTest { lout.write(logData); //statics make it annoying to test things correctly - CommitLog.instance.recover(new File[]{ logFile }); //CASSANDRA-1119 / CASSANDRA-1179 throw on failure*/ - } - } - - @Test - public void testVersions() - { - Assert.assertTrue(CommitLogDescriptor.isValid("CommitLog-1340512736956320000.log")); - Assert.assertTrue(CommitLogDescriptor.isValid("CommitLog-2-1340512736956320000.log")); - Assert.assertFalse(CommitLogDescriptor.isValid("CommitLog--1340512736956320000.log")); - Assert.assertFalse(CommitLogDescriptor.isValid("CommitLog--2-1340512736956320000.log")); - Assert.assertFalse(CommitLogDescriptor.isValid("CommitLog-2-1340512736956320000-123.log")); - - Assert.assertEquals(1340512736956320000L, CommitLogDescriptor.fromFileName("CommitLog-2-1340512736956320000.log").id); - - Assert.assertEquals(MessagingService.current_version, new CommitLogDescriptor(1340512736956320000L, null).getMessagingVersion()); - String newCLName = "CommitLog-" + CommitLogDescriptor.current_version + "-1340512736956320000.log"; - Assert.assertEquals(MessagingService.current_version, CommitLogDescriptor.fromFileName(newCLName).getMessagingVersion()); - } - - @Test - public void testTruncateWithoutSnapshot() throws IOException - { - CommitLog.instance.resetUnsafe(true); - boolean prev = DatabaseDescriptor.isAutoSnapshot(); - DatabaseDescriptor.setAutoSnapshot(false); - ColumnFamilyStore cfs1 = Keyspace.open(KEYSPACE1).getColumnFamilyStore("Standard1"); - ColumnFamilyStore cfs2 = Keyspace.open(KEYSPACE1).getColumnFamilyStore("Standard2"); - - final Mutation rm1 = new Mutation(KEYSPACE1, bytes("k")); - rm1.add("Standard1", Util.cellname("c1"), ByteBuffer.allocate(100), 0); - rm1.apply(); - cfs1.truncateBlocking(); - DatabaseDescriptor.setAutoSnapshot(prev); - final Mutation rm2 = new Mutation(KEYSPACE1, bytes("k")); - rm2.add("Standard2", Util.cellname("c1"), ByteBuffer.allocate(DatabaseDescriptor.getCommitLogSegmentSize() / 4), 0); - - for (int i = 0 ; i < 5 ; i++) - CommitLog.instance.add(rm2); - - Assert.assertEquals(2, CommitLog.instance.activeSegments()); - ReplayPosition position = CommitLog.instance.getContext(); - for (Keyspace ks : Keyspace.system()) - for (ColumnFamilyStore syscfs : ks.getColumnFamilyStores()) - CommitLog.instance.discardCompletedSegments(syscfs.metadata.cfId, position); - CommitLog.instance.discardCompletedSegments(cfs2.metadata.cfId, position); - Assert.assertEquals(1, CommitLog.instance.activeSegments()); - } - - @Test - public void testTruncateWithoutSnapshotNonDurable() throws IOException - { - CommitLog.instance.resetUnsafe(true); - boolean prevAutoSnapshot = DatabaseDescriptor.isAutoSnapshot(); - DatabaseDescriptor.setAutoSnapshot(false); - Keyspace notDurableKs = Keyspace.open(KEYSPACE2); - Assert.assertFalse(notDurableKs.metadata.durableWrites); - ColumnFamilyStore cfs = notDurableKs.getColumnFamilyStore("Standard1"); - CellNameType type = notDurableKs.getColumnFamilyStore("Standard1").getComparator(); - Mutation rm; - DecoratedKey dk = Util.dk("key1"); - - // add data - rm = new Mutation(KEYSPACE2, dk.getKey()); - rm.add("Standard1", Util.cellname("Column1"), ByteBufferUtil.bytes("abcd"), 0); - rm.apply(); - - ReadCommand command = new SliceByNamesReadCommand(KEYSPACE2, dk.getKey(), "Standard1", System.currentTimeMillis(), new NamesQueryFilter(FBUtilities.singleton(Util.cellname("Column1"), type))); - Row row = command.getRow(notDurableKs); - Cell col = row.cf.getColumn(Util.cellname("Column1")); - Assert.assertEquals(col.value(), ByteBuffer.wrap("abcd".getBytes())); - cfs.truncateBlocking(); - DatabaseDescriptor.setAutoSnapshot(prevAutoSnapshot); - row = command.getRow(notDurableKs); - Assert.assertEquals(null, row.cf); - } - - private void testDescriptorPersistence(CommitLogDescriptor desc) throws IOException - { - ByteBuffer buf = ByteBuffer.allocate(1024); - CommitLogDescriptor.writeHeader(buf, desc); - long length = buf.position(); - // Put some extra data in the stream. - buf.putDouble(0.1); - buf.flip(); - FileDataInput input = new ByteBufferDataInput(buf, "input", 0, 0); - CommitLogDescriptor read = CommitLogDescriptor.readHeader(input); - Assert.assertEquals("Descriptor length", length, input.getFilePointer()); - Assert.assertEquals("Descriptors", desc, read); - } - - @Test - public void testDescriptorPersistence() throws IOException - { - testDescriptorPersistence(new CommitLogDescriptor(11, null)); - testDescriptorPersistence(new CommitLogDescriptor(CommitLogDescriptor.VERSION_21, 13, null)); - testDescriptorPersistence(new CommitLogDescriptor(CommitLogDescriptor.VERSION_22, 15, null)); - testDescriptorPersistence(new CommitLogDescriptor(CommitLogDescriptor.VERSION_22, 17, new ParameterizedClass("LZ4Compressor", null))); - testDescriptorPersistence(new CommitLogDescriptor(CommitLogDescriptor.VERSION_22, 19, - new ParameterizedClass("StubbyCompressor", ImmutableMap.of("parameter1", "value1", "flag2", "55", "argument3", "null")))); - } - - @Test - public void testDescriptorInvalidParametersSize() throws IOException - { - Map params = new HashMap<>(); - for (int i=0; i<65535; ++i) - params.put("key"+i, Integer.toString(i, 16)); - try { - CommitLogDescriptor desc = new CommitLogDescriptor(CommitLogDescriptor.VERSION_22, - 21, - new ParameterizedClass("LZ4Compressor", params)); - ByteBuffer buf = ByteBuffer.allocate(1024000); - CommitLogDescriptor.writeHeader(buf, desc); - Assert.fail("Parameter object too long should fail on writing descriptor."); - } catch (ConfigurationException e) - { - // correct path + CommitLog.instance.recover(new File[]{ logFile }); //CASSANDRA-1119 / CASSANDRA-1179 throw on failure } } } + diff --git a/test/unit/org/apache/cassandra/db/CounterCacheTest.java b/test/unit/org/apache/cassandra/db/CounterCacheTest.java index 71f8b20336..035d8f81ce 100644 --- a/test/unit/org/apache/cassandra/db/CounterCacheTest.java +++ b/test/unit/org/apache/cassandra/db/CounterCacheTest.java @@ -19,13 +19,16 @@ package org.apache.cassandra.db; import java.util.concurrent.ExecutionException; +import org.apache.cassandra.config.CFMetaData; +import org.apache.cassandra.config.ColumnDefinition; +import org.apache.cassandra.utils.ByteBufferUtil; import org.junit.AfterClass; import org.junit.BeforeClass; import org.junit.Test; import org.apache.cassandra.SchemaLoader; import org.apache.cassandra.config.KSMetaData; -import org.apache.cassandra.db.marshal.CounterColumnType; +import org.apache.cassandra.db.marshal.*; import org.apache.cassandra.exceptions.ConfigurationException; import org.apache.cassandra.exceptions.WriteTimeoutException; import org.apache.cassandra.locator.SimpleStrategy; @@ -35,22 +38,28 @@ import org.apache.cassandra.utils.FBUtilities; import static org.junit.Assert.assertEquals; import static org.junit.Assert.assertNull; -import static org.apache.cassandra.Util.cellname; import static org.apache.cassandra.utils.ByteBufferUtil.bytes; public class CounterCacheTest { private static final String KEYSPACE1 = "CounterCacheTest"; - private static final String CF = "Counter1"; + private static final String COUNTER1 = "Counter1"; @BeforeClass public static void defineSchema() throws ConfigurationException { SchemaLoader.prepareServer(); + + CFMetaData counterTable = CFMetaData.Builder.create(KEYSPACE1, COUNTER1, false, true, true) + .addPartitionKey("key", Int32Type.instance) + .addClusteringColumn("name", Int32Type.instance) + .addRegularColumn("c", CounterColumnType.instance) + .build(); + SchemaLoader.createKeyspace(KEYSPACE1, - SimpleStrategy.class, - KSMetaData.optsWithRF(1), - SchemaLoader.standardCFMD(KEYSPACE1, CF).defaultValidator(CounterColumnType.instance)); + SimpleStrategy.class, + KSMetaData.optsWithRF(1), + counterTable); } @AfterClass @@ -62,38 +71,42 @@ public class CounterCacheTest @Test public void testReadWrite() { - ColumnFamilyStore cfs = Keyspace.open(KEYSPACE1).getColumnFamilyStore(CF); + ColumnFamilyStore cfs = Keyspace.open(KEYSPACE1).getColumnFamilyStore(COUNTER1); CacheService.instance.invalidateCounterCache(); + Clustering c1 = CBuilder.create(cfs.metadata.comparator).add(ByteBufferUtil.bytes(1)).build(); + Clustering c2 = CBuilder.create(cfs.metadata.comparator).add(ByteBufferUtil.bytes(2)).build(); + ColumnDefinition cd = cfs.metadata.getColumnDefinition(ByteBufferUtil.bytes("c")); + assertEquals(0, CacheService.instance.counterCache.size()); - assertNull(cfs.getCachedCounter(bytes(1), cellname(1))); - assertNull(cfs.getCachedCounter(bytes(1), cellname(2))); - assertNull(cfs.getCachedCounter(bytes(2), cellname(1))); - assertNull(cfs.getCachedCounter(bytes(2), cellname(2))); + assertNull(cfs.getCachedCounter(bytes(1), c1, cd, null)); + assertNull(cfs.getCachedCounter(bytes(1), c2, cd, null)); + assertNull(cfs.getCachedCounter(bytes(2), c1, cd, null)); + assertNull(cfs.getCachedCounter(bytes(2), c2, cd, null)); - cfs.putCachedCounter(bytes(1), cellname(1), ClockAndCount.create(1L, 1L)); - cfs.putCachedCounter(bytes(1), cellname(2), ClockAndCount.create(1L, 2L)); - cfs.putCachedCounter(bytes(2), cellname(1), ClockAndCount.create(2L, 1L)); - cfs.putCachedCounter(bytes(2), cellname(2), ClockAndCount.create(2L, 2L)); + cfs.putCachedCounter(bytes(1), c1, cd, null, ClockAndCount.create(1L, 1L)); + cfs.putCachedCounter(bytes(1), c2, cd, null, ClockAndCount.create(1L, 2L)); + cfs.putCachedCounter(bytes(2), c1, cd, null, ClockAndCount.create(2L, 1L)); + cfs.putCachedCounter(bytes(2), c2, cd, null, ClockAndCount.create(2L, 2L)); - assertEquals(4, CacheService.instance.counterCache.size()); - assertEquals(ClockAndCount.create(1L, 1L), cfs.getCachedCounter(bytes(1), cellname(1))); - assertEquals(ClockAndCount.create(1L, 2L), cfs.getCachedCounter(bytes(1), cellname(2))); - assertEquals(ClockAndCount.create(2L, 1L), cfs.getCachedCounter(bytes(2), cellname(1))); - assertEquals(ClockAndCount.create(2L, 2L), cfs.getCachedCounter(bytes(2), cellname(2))); + assertEquals(ClockAndCount.create(1L, 1L), cfs.getCachedCounter(bytes(1), c1, cd, null)); + assertEquals(ClockAndCount.create(1L, 2L), cfs.getCachedCounter(bytes(1), c2, cd, null)); + assertEquals(ClockAndCount.create(2L, 1L), cfs.getCachedCounter(bytes(2), c1, cd, null)); + assertEquals(ClockAndCount.create(2L, 2L), cfs.getCachedCounter(bytes(2), c2, cd, null)); } @Test public void testSaveLoad() throws ExecutionException, InterruptedException, WriteTimeoutException { - ColumnFamilyStore cfs = Keyspace.open(KEYSPACE1).getColumnFamilyStore(CF); + ColumnFamilyStore cfs = Keyspace.open(KEYSPACE1).getColumnFamilyStore(COUNTER1); CacheService.instance.invalidateCounterCache(); - ColumnFamily cells = ArrayBackedSortedColumns.factory.create(cfs.metadata); - cells.addColumn(new BufferCounterUpdateCell(cellname(1), 1L, FBUtilities.timestampMicros())); - cells.addColumn(new BufferCounterUpdateCell(cellname(2), 2L, FBUtilities.timestampMicros())); - new CounterMutation(new Mutation(KEYSPACE1, bytes(1), cells), ConsistencyLevel.ONE).apply(); - new CounterMutation(new Mutation(KEYSPACE1, bytes(2), cells), ConsistencyLevel.ONE).apply(); + new CounterMutation(new RowUpdateBuilder(cfs.metadata, 0, bytes(1)).clustering(1).add("c", 1L).build(), ConsistencyLevel.ONE).apply(); + new CounterMutation(new RowUpdateBuilder(cfs.metadata, 0, bytes(1)).clustering(2).add("c", 2L).build(), ConsistencyLevel.ONE).apply(); + new CounterMutation(new RowUpdateBuilder(cfs.metadata, 0, bytes(2)).clustering(1).add("c", 1L).build(), ConsistencyLevel.ONE).apply(); + new CounterMutation(new RowUpdateBuilder(cfs.metadata, 0, bytes(2)).clustering(2).add("c", 2L).build(), ConsistencyLevel.ONE).apply(); + + assertEquals(4, CacheService.instance.counterCache.size()); // flush the counter cache and invalidate CacheService.instance.counterCache.submitWrite(Integer.MAX_VALUE).get(); @@ -103,9 +116,14 @@ public class CounterCacheTest // load from cache and validate CacheService.instance.counterCache.loadSaved(cfs); assertEquals(4, CacheService.instance.counterCache.size()); - assertEquals(ClockAndCount.create(1L, 1L), cfs.getCachedCounter(bytes(1), cellname(1))); - assertEquals(ClockAndCount.create(1L, 2L), cfs.getCachedCounter(bytes(1), cellname(2))); - assertEquals(ClockAndCount.create(1L, 1L), cfs.getCachedCounter(bytes(2), cellname(1))); - assertEquals(ClockAndCount.create(1L, 2L), cfs.getCachedCounter(bytes(2), cellname(2))); + + Clustering c1 = CBuilder.create(cfs.metadata.comparator).add(ByteBufferUtil.bytes(1)).build(); + Clustering c2 = CBuilder.create(cfs.metadata.comparator).add(ByteBufferUtil.bytes(2)).build(); + ColumnDefinition cd = cfs.metadata.getColumnDefinition(ByteBufferUtil.bytes("c")); + + assertEquals(ClockAndCount.create(1L, 1L), cfs.getCachedCounter(bytes(1), c1, cd, null)); + assertEquals(ClockAndCount.create(1L, 2L), cfs.getCachedCounter(bytes(1), c2, cd, null)); + assertEquals(ClockAndCount.create(1L, 1L), cfs.getCachedCounter(bytes(2), c1, cd, null)); + assertEquals(ClockAndCount.create(1L, 2L), cfs.getCachedCounter(bytes(2), c2, cd, null)); } } diff --git a/test/unit/org/apache/cassandra/db/CounterCellTest.java b/test/unit/org/apache/cassandra/db/CounterCellTest.java index 8d75b9ac3a..025cb3cf77 100644 --- a/test/unit/org/apache/cassandra/db/CounterCellTest.java +++ b/test/unit/org/apache/cassandra/db/CounterCellTest.java @@ -18,26 +18,28 @@ */ package org.apache.cassandra.db; -import java.security.MessageDigest; - -import java.io.ByteArrayInputStream; -import java.io.DataInputStream; -import java.io.IOException; import java.nio.ByteBuffer; +import java.security.MessageDigest; import java.util.Arrays; +import org.junit.AfterClass; import org.junit.Assert; +import org.junit.BeforeClass; import org.junit.Test; -import org.apache.cassandra.Util; -import org.apache.cassandra.db.composites.CellNameType; -import org.apache.cassandra.db.composites.SimpleDenseCellNameType; +import org.apache.cassandra.SchemaLoader; +import org.apache.cassandra.config.ColumnDefinition; +import org.apache.cassandra.config.KSMetaData; +import org.apache.cassandra.db.rows.AbstractCell; +import org.apache.cassandra.db.rows.Cell; +import org.apache.cassandra.db.rows.CellPath; +import org.apache.cassandra.db.rows.Cells; import org.apache.cassandra.db.context.CounterContext; -import org.apache.cassandra.db.marshal.UTF8Type; -import org.apache.cassandra.io.util.DataOutputBuffer; +import org.apache.cassandra.exceptions.ConfigurationException; +import org.apache.cassandra.locator.SimpleStrategy; import org.apache.cassandra.utils.*; -import static org.apache.cassandra.Util.cellname; +import static org.junit.Assert.*; import static org.apache.cassandra.db.context.CounterContext.ContextState; public class CounterCellTest @@ -50,6 +52,27 @@ public class CounterCellTest private static final int stepLength; + private static final String KEYSPACE1 = "CounterCacheTest"; + private static final String COUNTER1 = "Counter1"; + private static final String STANDARD1 = "Standard1"; + + @BeforeClass + public static void defineSchema() throws ConfigurationException + { + SchemaLoader.prepareServer(); + SchemaLoader.createKeyspace(KEYSPACE1, + SimpleStrategy.class, + KSMetaData.optsWithRF(1), + SchemaLoader.standardCFMD(KEYSPACE1, STANDARD1), + SchemaLoader.counterCFMD(KEYSPACE1, COUNTER1)); + } + + @AfterClass + public static void cleanup() + { + SchemaLoader.cleanupSavedCaches(); + } + static { idLength = CounterId.LENGTH; @@ -59,244 +82,243 @@ public class CounterCellTest stepLength = idLength + clockLength + countLength; } + private class TestCounterCell extends AbstractCell + { + private final ColumnDefinition column; + private final ByteBuffer value; + private final LivenessInfo info; + + private TestCounterCell(ColumnDefinition column, ByteBuffer value, LivenessInfo info) + { + this.column = column; + this.value = value; + this.info = info.takeAlias(); + } + + public ColumnDefinition column() + { + return column; + } + + public boolean isCounterCell() + { + return true; + } + + public ByteBuffer value() + { + return value; + } + + public LivenessInfo livenessInfo() + { + return info; + } + + public CellPath path() + { + return null; + } + } + @Test public void testCreate() { + ColumnFamilyStore cfs = Keyspace.open(KEYSPACE1).getColumnFamilyStore(COUNTER1); long delta = 3L; - CounterCell cell = new BufferCounterCell(Util.cellname("x"), - CounterContext.instance().createLocal(delta), - 1L, - Long.MIN_VALUE); - Assert.assertEquals(delta, cell.total()); - Assert.assertEquals(1, cell.value().getShort(0)); - Assert.assertEquals(0, cell.value().getShort(2)); + TestCounterCell cell = createLegacyCounterCell(cfs, ByteBufferUtil.bytes("c1"), delta, 1, 0, 0); + + assertEquals(delta, CounterContext.instance().total(cell.value())); + assertEquals(1, cell.value().getShort(0)); + assertEquals(0, cell.value().getShort(2)); Assert.assertTrue(CounterId.wrap(cell.value(), 4).isLocalId()); - Assert.assertEquals(1L, cell.value().getLong(4 + idLength)); - Assert.assertEquals(delta, cell.value().getLong(4 + idLength + clockLength)); + assertEquals(1L, cell.value().getLong(4 + idLength)); + assertEquals(delta, cell.value().getLong(4 + idLength + clockLength)); + + } + + private TestCounterCell createLegacyCounterCell(ColumnFamilyStore cfs, + ByteBuffer colName, + long count, + long ts, + int ttl, + int localDeletion) + { + ColumnDefinition cDef = cfs.metadata.getColumnDefinition(colName); + ByteBuffer val = CounterContext.instance().createLocal(count); + LivenessInfo li = new SimpleLivenessInfo(ts, ttl, localDeletion); + return new TestCounterCell(cDef, val, li); + } + + private TestCounterCell createCounterCell(ColumnFamilyStore cfs, + ByteBuffer colName, + CounterId id, + long count, + long ts, + int ttl, + int localDeletion) + { + ColumnDefinition cDef = cfs.metadata.getColumnDefinition(colName); + ByteBuffer val = CounterContext.instance().createGlobal(id, ts, count); + LivenessInfo li = new SimpleLivenessInfo(ts, ttl, localDeletion); + return new TestCounterCell(cDef, val, li); + } + + private TestCounterCell createCounterCellFromContext(ColumnFamilyStore cfs, + ByteBuffer colName, + ContextState context, + long ts, + int ttl, + int localDeletion) + { + ColumnDefinition cDef = cfs.metadata.getColumnDefinition(colName); + LivenessInfo li = new SimpleLivenessInfo(ts, ttl, localDeletion); + return new TestCounterCell(cDef, context.context, li); } @Test public void testReconcile() { - Cell left; - Cell right; - Cell reconciled; + ColumnFamilyStore cfs = Keyspace.open(KEYSPACE1).getColumnFamilyStore(COUNTER1); + ColumnDefinition cDef = Keyspace.open(KEYSPACE1).getColumnFamilyStore(STANDARD1).metadata.getColumnDefinition(ByteBufferUtil.bytes("val")); + ByteBuffer col = ByteBufferUtil.bytes("c1"); - ByteBuffer context; + AbstractCell left; + AbstractCell right; - // tombstone + tombstone - left = new BufferDeletedCell(cellname("x"), 1, 1L); - right = new BufferDeletedCell(cellname("x"), 2, 2L); + // both deleted, diff deletion time, same ts + left = createLegacyCounterCell(cfs, col, 1, 2, 0, 5); + right = createLegacyCounterCell(cfs, col, 1, 2, 0, 10); + assert Cells.reconcile(left, right, 10) == right; - assert left.reconcile(right).timestamp() == right.timestamp(); - assert right.reconcile(left).timestamp() == right.timestamp(); + // diff ts + right = createLegacyCounterCell(cfs, col, 1, 1, 0, 10); + assert Cells.reconcile(left, right, 10) == left; - // tombstone > live - left = new BufferDeletedCell(cellname("x"), 1, 2L); - right = BufferCounterCell.createLocal(cellname("x"), 0L, 1L, Long.MIN_VALUE); + // < tombstone + left = new CellTest.TestCell(cDef, ByteBufferUtil.bytes(5L), SimpleLivenessInfo.forDeletion(6, 6)); + right = createLegacyCounterCell(cfs, col, 1, 5, 0, 5); + assert Cells.reconcile(left, right, 10) == left; - assert left.reconcile(right) == left; + // > tombstone + left = new CellTest.TestCell(cDef, ByteBufferUtil.bytes(5L), SimpleLivenessInfo.forDeletion(1, 1)); + right = createLegacyCounterCell(cfs, col, 1, 5, 0, Integer.MAX_VALUE); + assert Cells.reconcile(left, right, 10) == left; - // tombstone < live last delete - left = new BufferDeletedCell(cellname("x"), 1, 1L); - right = BufferCounterCell.createLocal(cellname("x"), 0L, 4L, 2L); - - assert left.reconcile(right) == left; - - // tombstone == live last delete - left = new BufferDeletedCell(cellname("x"), 1, 2L); - right = BufferCounterCell.createLocal(cellname("x"), 0L, 4L, 2L); - - assert left.reconcile(right) == left; - - // tombstone > live last delete - left = new BufferDeletedCell(cellname("x"), 1, 4L); - right = BufferCounterCell.createLocal(cellname("x"), 0L, 9L, 1L); - - assert left.reconcile(right) == left; - - // live < tombstone - left = BufferCounterCell.createLocal(cellname("x"), 0L, 1L, Long.MIN_VALUE); - right = new BufferDeletedCell(cellname("x"), 1, 2L); - - assert left.reconcile(right) == right; - - // live last delete > tombstone - left = BufferCounterCell.createLocal(cellname("x"), 0L, 4L, 2L); - right = new BufferDeletedCell(cellname("x"), 1, 1L); - - assert left.reconcile(right) == right; - - // live last delete == tombstone - left = BufferCounterCell.createLocal(cellname("x"), 0L, 4L, 2L); - right = new BufferDeletedCell(cellname("x"), 1, 2L); - - assert left.reconcile(right) == right; - - // live last delete < tombstone - left = BufferCounterCell.createLocal(cellname("x"), 0L, 9L, 1L); - right = new BufferDeletedCell(cellname("x"), 1, 4L); - - assert left.reconcile(right) == right; - - // live < live last delete - left = new BufferCounterCell(cellname("x"), cc.createRemote(CounterId.fromInt(1), 2L, 3L), 1L, Long.MIN_VALUE); - right = new BufferCounterCell(cellname("x"), cc.createRemote(CounterId.fromInt(1), 1L, 1L), 4L, 3L); - - assert left.reconcile(right) == right; - - // live last delete > live - left = new BufferCounterCell(cellname("x"), cc.createRemote(CounterId.fromInt(1), 2L, 3L), 6L, 5L); - right = new BufferCounterCell(cellname("x"), cc.createRemote(CounterId.fromInt(1), 1L, 1L), 4L, 3L); - - assert left.reconcile(right) == left; + // == tombstone + left = new CellTest.TestCell(cDef, ByteBufferUtil.bytes(5L), SimpleLivenessInfo.forDeletion(8, 8)); + right = createLegacyCounterCell(cfs, col, 1, 8, 0, Integer.MAX_VALUE); + assert Cells.reconcile(left, right, 10) == left; // live + live - left = new BufferCounterCell(cellname("x"), cc.createRemote(CounterId.fromInt(1), 1L, 1L), 4L, Long.MIN_VALUE); - right = new BufferCounterCell(cellname("x"), cc.createRemote(CounterId.fromInt(1), 2L, 3L), 1L, Long.MIN_VALUE); + left = createLegacyCounterCell(cfs, col, 1, 2, 0, Integer.MAX_VALUE); + right = createLegacyCounterCell(cfs, col, 3, 5, 0, Integer.MAX_VALUE); + Cell reconciled = Cells.reconcile(left, right, 10); + assertEquals(CounterContext.instance().total(reconciled.value()), 4); + assertEquals(reconciled.livenessInfo().timestamp(), 5L); - reconciled = left.reconcile(right); - assert reconciled.name().equals(left.name()); - assert ((CounterCell)reconciled).total() == 3L; - assert reconciled.timestamp() == 4L; + // Add, don't change TS + Cell addTen = createLegacyCounterCell(cfs, col, 10, 4, 0, Integer.MAX_VALUE); + reconciled = Cells.reconcile(reconciled, addTen, 10); + assertEquals(CounterContext.instance().total(reconciled.value()), 14); + assertEquals(reconciled.livenessInfo().timestamp(), 5L); - left = reconciled; - right = new BufferCounterCell(cellname("x"), cc.createRemote(CounterId.fromInt(2), 1L, 5L), 2L, Long.MIN_VALUE); + // Add w/new TS + Cell addThree = createLegacyCounterCell(cfs, col, 3, 7, 0, Integer.MAX_VALUE); + reconciled = Cells.reconcile(reconciled, addThree, 10); + assertEquals(CounterContext.instance().total(reconciled.value()), 17); + assertEquals(reconciled.livenessInfo().timestamp(), 7L); - reconciled = left.reconcile(right); - assert reconciled.name().equals(left.name()); - assert ((CounterCell)reconciled).total() == 8L; - assert reconciled.timestamp() == 4L; + // Confirm no deletion time + assert reconciled.livenessInfo().localDeletionTime() == Integer.MAX_VALUE; - left = reconciled; - right = new BufferCounterCell(cellname("x"), cc.createRemote(CounterId.fromInt(2), 2L, 2L), 6L, Long.MIN_VALUE); - - reconciled = left.reconcile(right); - assert reconciled.name().equals(left.name()); - assert ((CounterCell)reconciled).total() == 5L; - assert reconciled.timestamp() == 6L; - - context = reconciled.value(); - int hd = 2; // header - assert hd + 2 * stepLength == context.remaining(); - - assert Util.equalsCounterId(CounterId.fromInt(1), context, hd); - assert 2L == context.getLong(hd + idLength); - assert 3L == context.getLong(hd + idLength + clockLength); - - assert Util.equalsCounterId(CounterId.fromInt(2), context, hd + stepLength); - assert 2L == context.getLong(hd + stepLength + idLength); - assert 2L == context.getLong(hd + stepLength + idLength + clockLength); - - assert ((CounterCell)reconciled).timestampOfLastDelete() == Long.MIN_VALUE; + Cell deleted = createLegacyCounterCell(cfs, col, 2, 8, 0, 8); + reconciled = Cells.reconcile(reconciled, deleted, 10); + assertEquals(2, CounterContext.instance().total(reconciled.value())); + assertEquals(reconciled.livenessInfo().timestamp(), 8L); + assert reconciled.livenessInfo().localDeletionTime() == 8; } @Test public void testDiff() { - ContextState left; - ContextState right; + ColumnFamilyStore cfs = Keyspace.open(KEYSPACE1).getColumnFamilyStore(COUNTER1); + ByteBuffer col = ByteBufferUtil.bytes("c1"); - CounterCell leftCell; - CounterCell rightCell; + TestCounterCell leftCell; + TestCounterCell rightCell; + + // Equal count + leftCell = createLegacyCounterCell(cfs, col, 2, 2, 0, Integer.MAX_VALUE); + rightCell = createLegacyCounterCell(cfs, col, 2, 1, 0, Integer.MAX_VALUE); + assertEquals(CounterContext.Relationship.EQUAL, CounterContext.instance().diff(leftCell.value(), rightCell.value())); + + // Non-equal count + leftCell = createLegacyCounterCell(cfs, col, 1, 2, 0, Integer.MAX_VALUE); + rightCell = createLegacyCounterCell(cfs, col, 2, 1, 0, Integer.MAX_VALUE); + assertEquals(CounterContext.Relationship.DISJOINT, CounterContext.instance().diff(leftCell.value(), rightCell.value())); // timestamp - leftCell = BufferCounterCell.createLocal(cellname("x"), 0, 1L, Long.MIN_VALUE); - rightCell = BufferCounterCell.createLocal(cellname("x"), 0, 2L, Long.MIN_VALUE); + CounterId id = CounterId.generate(); + leftCell = createCounterCell(cfs, col, id, 2, 2, 0, Integer.MAX_VALUE); + rightCell = createCounterCell(cfs, col, id, 2, 1, 0, Integer.MAX_VALUE); + assertEquals(CounterContext.Relationship.GREATER_THAN, CounterContext.instance().diff(leftCell.value(), rightCell.value())); - assert rightCell == leftCell.diff(rightCell); - assert null == rightCell.diff(leftCell); + ContextState leftContext; + ContextState rightContext; - // timestampOfLastDelete - leftCell = BufferCounterCell.createLocal(cellname("x"), 0, 1L, 1L); - rightCell = BufferCounterCell.createLocal(cellname("x"), 0, 1L, 2L); + // Equal based on context w/shards etc + leftContext = ContextState.allocate(0, 0, 3); + leftContext.writeRemote(CounterId.fromInt(3), 3L, 0L); + leftContext.writeRemote(CounterId.fromInt(6), 2L, 0L); + leftContext.writeRemote(CounterId.fromInt(9), 1L, 0L); + rightContext = ContextState.wrap(ByteBufferUtil.clone(leftContext.context)); - assert rightCell == leftCell.diff(rightCell); - assert null == rightCell.diff(leftCell); - - // equality: equal nodes, all counts same - left = ContextState.allocate(0, 0, 3); - left.writeRemote(CounterId.fromInt(3), 3L, 0L); - left.writeRemote(CounterId.fromInt(6), 2L, 0L); - left.writeRemote(CounterId.fromInt(9), 1L, 0L); - right = ContextState.wrap(ByteBufferUtil.clone(left.context)); - - leftCell = new BufferCounterCell(cellname("x"), left.context, 1L); - rightCell = new BufferCounterCell(cellname("x"), right.context, 1L); - assert leftCell.diff(rightCell) == null; + leftCell = createCounterCellFromContext(cfs, col, leftContext, 1, 0, Integer.MAX_VALUE); + rightCell = createCounterCellFromContext(cfs, col, rightContext, 1, 0, Integer.MAX_VALUE); + assertEquals(CounterContext.Relationship.EQUAL, CounterContext.instance().diff(leftCell.value, rightCell.value)); // greater than: left has superset of nodes (counts equal) - left = ContextState.allocate(0, 0, 4); - left.writeRemote(CounterId.fromInt(3), 3L, 0L); - left.writeRemote(CounterId.fromInt(6), 2L, 0L); - left.writeRemote(CounterId.fromInt(9), 1L, 0L); - left.writeRemote(CounterId.fromInt(12), 0L, 0L); + leftContext = ContextState.allocate(0, 0, 4); + leftContext.writeRemote(CounterId.fromInt(3), 3L, 0L); + leftContext.writeRemote(CounterId.fromInt(6), 2L, 0L); + leftContext.writeRemote(CounterId.fromInt(9), 1L, 0L); + leftContext.writeRemote(CounterId.fromInt(12), 0L, 0L); - right = ContextState.allocate(0, 0, 3); - right.writeRemote(CounterId.fromInt(3), 3L, 0L); - right.writeRemote(CounterId.fromInt(6), 2L, 0L); - right.writeRemote(CounterId.fromInt(9), 1L, 0L); + rightContext = ContextState.allocate(0, 0, 3); + rightContext.writeRemote(CounterId.fromInt(3), 3L, 0L); + rightContext.writeRemote(CounterId.fromInt(6), 2L, 0L); + rightContext.writeRemote(CounterId.fromInt(9), 1L, 0L); - leftCell = new BufferCounterCell(cellname("x"), left.context, 1L); - rightCell = new BufferCounterCell(cellname("x"), right.context, 1L); - assert leftCell.diff(rightCell) == null; - - // less than: right has subset of nodes (counts equal) - assert leftCell == rightCell.diff(leftCell); + leftCell = createCounterCellFromContext(cfs, col, leftContext, 1, 0, Integer.MAX_VALUE); + rightCell = createCounterCellFromContext(cfs, col, rightContext, 1, 0, Integer.MAX_VALUE); + assertEquals(CounterContext.Relationship.GREATER_THAN, CounterContext.instance().diff(leftCell.value, rightCell.value)); + assertEquals(CounterContext.Relationship.LESS_THAN, CounterContext.instance().diff(rightCell.value, leftCell.value)); // disjoint: right and left have disjoint node sets - left = ContextState.allocate(0, 0, 3); - left.writeRemote(CounterId.fromInt(3), 1L, 0L); - left.writeRemote(CounterId.fromInt(4), 1L, 0L); - left.writeRemote(CounterId.fromInt(9), 1L, 0L); + leftContext = ContextState.allocate(0, 0, 3); + leftContext.writeRemote(CounterId.fromInt(3), 1L, 0L); + leftContext.writeRemote(CounterId.fromInt(4), 1L, 0L); + leftContext.writeRemote(CounterId.fromInt(9), 1L, 0L); - right = ContextState.allocate(0, 0, 3); - right.writeRemote(CounterId.fromInt(3), 1L, 0L); - right.writeRemote(CounterId.fromInt(6), 1L, 0L); - right.writeRemote(CounterId.fromInt(9), 1L, 0L); + rightContext = ContextState.allocate(0, 0, 3); + rightContext.writeRemote(CounterId.fromInt(3), 1L, 0L); + rightContext.writeRemote(CounterId.fromInt(6), 1L, 0L); + rightContext.writeRemote(CounterId.fromInt(9), 1L, 0L); - leftCell = new BufferCounterCell(cellname("x"), left.context, 1L); - rightCell = new BufferCounterCell(cellname("x"), right.context, 1L); - assert rightCell == leftCell.diff(rightCell); - assert leftCell == rightCell.diff(leftCell); - } - - @Test - public void testSerializeDeserialize() throws IOException - { - CounterContext.ContextState state = CounterContext.ContextState.allocate(0, 2, 2); - state.writeRemote(CounterId.fromInt(1), 4L, 4L); - state.writeLocal(CounterId.fromInt(2), 4L, 4L); - state.writeRemote(CounterId.fromInt(3), 4L, 4L); - state.writeLocal(CounterId.fromInt(4), 4L, 4L); - - CellNameType type = new SimpleDenseCellNameType(UTF8Type.instance); - CounterCell original = new BufferCounterCell(cellname("x"), state.context, 1L); - byte[] serialized; - try (DataOutputBuffer bufOut = new DataOutputBuffer()) - { - type.columnSerializer().serialize(original, bufOut); - serialized = bufOut.getData(); - } - - - ByteArrayInputStream bufIn = new ByteArrayInputStream(serialized, 0, serialized.length); - CounterCell deserialized = (CounterCell) type.columnSerializer().deserialize(new DataInputStream(bufIn)); - Assert.assertEquals(original, deserialized); - - bufIn = new ByteArrayInputStream(serialized, 0, serialized.length); - CounterCell deserializedOnRemote = (CounterCell) type.columnSerializer().deserialize(new DataInputStream(bufIn), ColumnSerializer.Flag.FROM_REMOTE); - Assert.assertEquals(deserializedOnRemote.name(), original.name()); - Assert.assertEquals(deserializedOnRemote.total(), original.total()); - Assert.assertEquals(deserializedOnRemote.value(), cc.clearAllLocal(original.value())); - Assert.assertEquals(deserializedOnRemote.timestamp(), deserialized.timestamp()); - Assert.assertEquals(deserializedOnRemote.timestampOfLastDelete(), deserialized.timestampOfLastDelete()); + leftCell = createCounterCellFromContext(cfs, col, leftContext, 1, 0, Integer.MAX_VALUE); + rightCell = createCounterCellFromContext(cfs, col, rightContext, 1, 0, Integer.MAX_VALUE); + assertEquals(CounterContext.Relationship.DISJOINT, CounterContext.instance().diff(leftCell.value, rightCell.value)); + assertEquals(CounterContext.Relationship.DISJOINT, CounterContext.instance().diff(rightCell.value, leftCell.value)); } @Test public void testUpdateDigest() throws Exception { + ColumnFamilyStore cfs = Keyspace.open(KEYSPACE1).getColumnFamilyStore(COUNTER1); + ByteBuffer col = ByteBufferUtil.bytes("c1"); + MessageDigest digest1 = MessageDigest.getInstance("md5"); MessageDigest digest2 = MessageDigest.getInstance("md5"); @@ -306,11 +328,14 @@ public class CounterCellTest state.writeRemote(CounterId.fromInt(3), 4L, 4L); state.writeLocal(CounterId.fromInt(4), 4L, 4L); - CounterCell original = new BufferCounterCell(cellname("x"), state.context, 1L); - CounterCell cleared = new BufferCounterCell(cellname("x"), cc.clearAllLocal(state.context), 1L); + TestCounterCell original = createCounterCellFromContext(cfs, col, state, 5, 0, Integer.MAX_VALUE); - original.updateDigest(digest1); - cleared.updateDigest(digest2); + ColumnDefinition cDef = cfs.metadata.getColumnDefinition(col); + LivenessInfo li = new SimpleLivenessInfo(5, 0, Integer.MAX_VALUE); + TestCounterCell cleared = new TestCounterCell(cDef, CounterContext.instance().clearAllLocal(state.context), li); + + CounterContext.instance().updateDigest(digest1, original.value); + CounterContext.instance().updateDigest(digest2, cleared.value); assert Arrays.equals(digest1.digest(), digest2.digest()); } diff --git a/test/unit/org/apache/cassandra/db/CounterMutationTest.java b/test/unit/org/apache/cassandra/db/CounterMutationTest.java index 0aa33c5060..4e67a7e641 100644 --- a/test/unit/org/apache/cassandra/db/CounterMutationTest.java +++ b/test/unit/org/apache/cassandra/db/CounterMutationTest.java @@ -17,25 +17,24 @@ */ package org.apache.cassandra.db; -import java.nio.ByteBuffer; - import org.junit.BeforeClass; import org.junit.Test; +import org.apache.cassandra.Util; +import org.apache.cassandra.config.ColumnDefinition; import org.apache.cassandra.config.KSMetaData; import org.apache.cassandra.SchemaLoader; +import org.apache.cassandra.db.rows.Row; import org.apache.cassandra.db.context.CounterContext; -import org.apache.cassandra.db.filter.QueryFilter; -import org.apache.cassandra.db.marshal.CounterColumnType; +import org.apache.cassandra.db.partitions.PartitionIterator; import org.apache.cassandra.exceptions.ConfigurationException; import org.apache.cassandra.exceptions.WriteTimeoutException; import org.apache.cassandra.locator.SimpleStrategy; -import org.apache.cassandra.utils.FBUtilities; +import org.apache.cassandra.utils.ByteBufferUtil; import static org.junit.Assert.assertEquals; import static org.junit.Assert.assertNull; -import static org.apache.cassandra.Util.cellname; import static org.apache.cassandra.Util.dk; import static org.apache.cassandra.utils.ByteBufferUtil.bytes; @@ -52,8 +51,8 @@ public class CounterMutationTest SchemaLoader.createKeyspace(KEYSPACE1, SimpleStrategy.class, KSMetaData.optsWithRF(1), - SchemaLoader.standardCFMD(KEYSPACE1, CF1).defaultValidator(CounterColumnType.instance), - SchemaLoader.standardCFMD(KEYSPACE1, CF2).defaultValidator(CounterColumnType.instance)); + SchemaLoader.counterCFMD(KEYSPACE1, CF1), + SchemaLoader.counterCFMD(KEYSPACE1, CF2)); } @Test @@ -61,28 +60,26 @@ public class CounterMutationTest { ColumnFamilyStore cfs = Keyspace.open(KEYSPACE1).getColumnFamilyStore(CF1); cfs.truncateBlocking(); + ColumnDefinition cDef = cfs.metadata.getColumnDefinition(ByteBufferUtil.bytes("val")); // Do the initial update (+1) - ColumnFamily cells = ArrayBackedSortedColumns.factory.create(cfs.metadata); - cells.addCounter(cellname(1), 1L); - new CounterMutation(new Mutation(KEYSPACE1, bytes(1), cells), ConsistencyLevel.ONE).apply(); - ColumnFamily current = cfs.getColumnFamily(QueryFilter.getIdentityFilter(dk(bytes(1)), CF1, System.currentTimeMillis())); - assertEquals(1L, CounterContext.instance().total(current.getColumn(cellname(1)).value())); + addAndCheck(cfs, 1, 1); // Make another increment (+2) - cells = ArrayBackedSortedColumns.factory.create(cfs.metadata); - cells.addCounter(cellname(1), 2L); - new CounterMutation(new Mutation(KEYSPACE1, bytes(1), cells), ConsistencyLevel.ONE).apply(); - current = cfs.getColumnFamily(QueryFilter.getIdentityFilter(dk(bytes(1)), CF1, System.currentTimeMillis())); - assertEquals(3L, CounterContext.instance().total(current.getColumn(cellname(1)).value())); + addAndCheck(cfs, 2, 3); // Decrement to 0 (-3) - cells = ArrayBackedSortedColumns.factory.create(cfs.metadata); - cells.addCounter(cellname(1), -3L); - new CounterMutation(new Mutation(KEYSPACE1, bytes(1), cells), ConsistencyLevel.ONE).apply(); - current = cfs.getColumnFamily(QueryFilter.getIdentityFilter(dk(bytes(1)), CF1, System.currentTimeMillis())); - assertEquals(0L, CounterContext.instance().total(current.getColumn(cellname(1)).value())); - assertEquals(ClockAndCount.create(3L, 0L), cfs.getCachedCounter(bytes(1), cellname(1))); + addAndCheck(cfs, -3, 0); + } + + private void addAndCheck(ColumnFamilyStore cfs, long toAdd, long expected) + { + ColumnDefinition cDef = cfs.metadata.getColumnDefinition(ByteBufferUtil.bytes("val")); + Mutation m = new RowUpdateBuilder(cfs.metadata, 5, "key1").clustering("cc").add("val", toAdd).build(); + new CounterMutation(m, ConsistencyLevel.ONE).apply(); + + Row row = Util.getOnlyRow(Util.cmd(cfs).includeRow("cc").columns("val").build()); + assertEquals(expected, CounterContext.instance().total(row.getCell(cDef).value())); } @Test @@ -92,74 +89,79 @@ public class CounterMutationTest cfs.truncateBlocking(); // Do the initial update (+1, -1) - ColumnFamily cells = ArrayBackedSortedColumns.factory.create(cfs.metadata); - cells.addCounter(cellname(1), 1L); - cells.addCounter(cellname(2), -1L); - new CounterMutation(new Mutation(KEYSPACE1, bytes(1), cells), ConsistencyLevel.ONE).apply(); - ColumnFamily current = cfs.getColumnFamily(QueryFilter.getIdentityFilter(dk(bytes(1)), CF1, System.currentTimeMillis())); - assertEquals(1L, CounterContext.instance().total(current.getColumn(cellname(1)).value())); - assertEquals(-1L, CounterContext.instance().total(current.getColumn(cellname(2)).value())); + addTwoAndCheck(cfs, 1L, 1L, -1L, -1L); // Make another increment (+2, -2) - cells = ArrayBackedSortedColumns.factory.create(cfs.metadata); - cells.addCounter(cellname(1), 2L); - cells.addCounter(cellname(2), -2L); - new CounterMutation(new Mutation(KEYSPACE1, bytes(1), cells), ConsistencyLevel.ONE).apply(); - current = cfs.getColumnFamily(QueryFilter.getIdentityFilter(dk(bytes(1)), CF1, System.currentTimeMillis())); - assertEquals(3L, CounterContext.instance().total(current.getColumn(cellname(1)).value())); + addTwoAndCheck(cfs, 2L, 3L, -2L, -3L); // Decrement to 0 (-3, +3) - cells = ArrayBackedSortedColumns.factory.create(cfs.metadata); - cells.addCounter(cellname(1), -3L); - cells.addCounter(cellname(2), 3L); - new CounterMutation(new Mutation(KEYSPACE1, bytes(1), cells), ConsistencyLevel.ONE).apply(); - current = cfs.getColumnFamily(QueryFilter.getIdentityFilter(dk(bytes(1)), CF1, System.currentTimeMillis())); - assertEquals(0L, CounterContext.instance().total(current.getColumn(cellname(1)).value())); - assertEquals(0L, CounterContext.instance().total(current.getColumn(cellname(2)).value())); + addTwoAndCheck(cfs, -3L, 0L, 3L, 0L); + } - // Check the caches, separately - assertEquals(ClockAndCount.create(3L, 0L), cfs.getCachedCounter(bytes(1), cellname(1))); - assertEquals(ClockAndCount.create(3L, 0L), cfs.getCachedCounter(bytes(1), cellname(2))); + private void addTwoAndCheck(ColumnFamilyStore cfs, long addOne, long expectedOne, long addTwo, long expectedTwo) + { + ColumnDefinition cDefOne = cfs.metadata.getColumnDefinition(ByteBufferUtil.bytes("val")); + ColumnDefinition cDefTwo = cfs.metadata.getColumnDefinition(ByteBufferUtil.bytes("val2")); + + Mutation m = new RowUpdateBuilder(cfs.metadata, 5, "key1") + .clustering("cc") + .add("val", addOne) + .add("val2", addTwo) + .build(); + new CounterMutation(m, ConsistencyLevel.ONE).apply(); + + Row row = Util.getOnlyRow(Util.cmd(cfs).includeRow("cc").columns("val", "val2").build()); + assertEquals(expectedOne, CounterContext.instance().total(row.getCell(cDefOne).value())); + assertEquals(expectedTwo, CounterContext.instance().total(row.getCell(cDefTwo).value())); } @Test public void testBatch() throws WriteTimeoutException { - ColumnFamilyStore cfs1 = Keyspace.open(KEYSPACE1).getColumnFamilyStore(CF1); - ColumnFamilyStore cfs2 = Keyspace.open(KEYSPACE1).getColumnFamilyStore(CF2); + ColumnFamilyStore cfsOne = Keyspace.open(KEYSPACE1).getColumnFamilyStore(CF1); + ColumnFamilyStore cfsTwo = Keyspace.open(KEYSPACE1).getColumnFamilyStore(CF2); - cfs1.truncateBlocking(); - cfs2.truncateBlocking(); + cfsOne.truncateBlocking(); + cfsTwo.truncateBlocking(); // Do the update (+1, -1), (+2, -2) - ColumnFamily cells1 = ArrayBackedSortedColumns.factory.create(cfs1.metadata); - cells1.addCounter(cellname(1), 1L); - cells1.addCounter(cellname(2), -1L); + Mutation batch = new Mutation(KEYSPACE1, Util.dk("key1")); + batch.add(new RowUpdateBuilder(cfsOne.metadata, 5, "key1") + .clustering("cc") + .add("val", 1L) + .add("val2", -1L) + .build().get(cfsOne.metadata)); - ColumnFamily cells2 = ArrayBackedSortedColumns.factory.create(cfs2.metadata); - cells2.addCounter(cellname(1), 2L); - cells2.addCounter(cellname(2), -2L); + batch.add(new RowUpdateBuilder(cfsTwo.metadata, 5, "key1") + .clustering("cc") + .add("val", 2L) + .add("val2", -2L) + .build().get(cfsTwo.metadata)); - Mutation mutation = new Mutation(KEYSPACE1, bytes(1)); - mutation.add(cells1); - mutation.add(cells2); + new CounterMutation(batch, ConsistencyLevel.ONE).apply(); - new CounterMutation(mutation, ConsistencyLevel.ONE).apply(); + ColumnDefinition c1cfs1 = cfsOne.metadata.getColumnDefinition(ByteBufferUtil.bytes("val")); + ColumnDefinition c2cfs1 = cfsOne.metadata.getColumnDefinition(ByteBufferUtil.bytes("val2")); - // Validate all values - ColumnFamily current1 = cfs1.getColumnFamily(QueryFilter.getIdentityFilter(dk(bytes(1)), CF1, System.currentTimeMillis())); - ColumnFamily current2 = cfs2.getColumnFamily(QueryFilter.getIdentityFilter(dk(bytes(1)), CF2, System.currentTimeMillis())); + Row row = Util.getOnlyRow(Util.cmd(cfsOne).includeRow("cc").columns("val", "val2").build()); + assertEquals(1L, CounterContext.instance().total(row.getCell(c1cfs1).value())); + assertEquals(-1L, CounterContext.instance().total(row.getCell(c2cfs1).value())); - assertEquals(1L, CounterContext.instance().total(current1.getColumn(cellname(1)).value())); - assertEquals(-1L, CounterContext.instance().total(current1.getColumn(cellname(2)).value())); - assertEquals(2L, CounterContext.instance().total(current2.getColumn(cellname(1)).value())); - assertEquals(-2L, CounterContext.instance().total(current2.getColumn(cellname(2)).value())); + ColumnDefinition c1cfs2 = cfsTwo.metadata.getColumnDefinition(ByteBufferUtil.bytes("val")); + ColumnDefinition c2cfs2 = cfsTwo.metadata.getColumnDefinition(ByteBufferUtil.bytes("val2")); + row = Util.getOnlyRow(Util.cmd(cfsTwo).includeRow("cc").columns("val", "val2").build()); + assertEquals(2L, CounterContext.instance().total(row.getCell(c1cfs2).value())); + assertEquals(-2L, CounterContext.instance().total(row.getCell(c2cfs2).value())); // Check the caches, separately - assertEquals(ClockAndCount.create(1L, 1L), cfs1.getCachedCounter(bytes(1), cellname(1))); - assertEquals(ClockAndCount.create(1L, -1L), cfs1.getCachedCounter(bytes(1), cellname(2))); - assertEquals(ClockAndCount.create(1L, 2L), cfs2.getCachedCounter(bytes(1), cellname(1))); - assertEquals(ClockAndCount.create(1L, -2L), cfs2.getCachedCounter(bytes(1), cellname(2))); + CBuilder cb = CBuilder.create(cfsOne.metadata.comparator); + cb.add("cc"); + + assertEquals(ClockAndCount.create(1L, 1L), cfsOne.getCachedCounter(Util.dk("key1").getKey(), cb.build(), c1cfs1, null)); + assertEquals(ClockAndCount.create(1L, -1L), cfsOne.getCachedCounter(Util.dk("key1").getKey(), cb.build(), c2cfs1, null)); + + assertEquals(ClockAndCount.create(1L, 2L), cfsTwo.getCachedCounter(Util.dk("key1").getKey(), cb.build(), c1cfs2, null)); + assertEquals(ClockAndCount.create(1L, -2L), cfsTwo.getCachedCounter(Util.dk("key1").getKey(), cb.build(), c2cfs2, null)); } @Test @@ -167,67 +169,57 @@ public class CounterMutationTest { ColumnFamilyStore cfs = Keyspace.open(KEYSPACE1).getColumnFamilyStore(CF1); cfs.truncateBlocking(); + ColumnDefinition cOne = cfs.metadata.getColumnDefinition(ByteBufferUtil.bytes("val")); + ColumnDefinition cTwo = cfs.metadata.getColumnDefinition(ByteBufferUtil.bytes("val2")); // Do the initial update (+1, -1) - ColumnFamily cells = ArrayBackedSortedColumns.factory.create(cfs.metadata); - cells.addCounter(cellname(1), 1L); - cells.addCounter(cellname(2), 1L); - new CounterMutation(new Mutation(KEYSPACE1, bytes(1), cells), ConsistencyLevel.ONE).apply(); - ColumnFamily current = cfs.getColumnFamily(QueryFilter.getIdentityFilter(dk(bytes(1)), CF1, System.currentTimeMillis())); - assertEquals(1L, CounterContext.instance().total(current.getColumn(cellname(1)).value())); - assertEquals(1L, CounterContext.instance().total(current.getColumn(cellname(2)).value())); + new CounterMutation( + new RowUpdateBuilder(cfs.metadata, 5, "key1") + .clustering("cc") + .add("val", 1L) + .add("val2", -1L) + .build(), + ConsistencyLevel.ONE).apply(); + + Row row = Util.getOnlyRow(Util.cmd(cfs).includeRow("cc").columns("val", "val2").build()); + assertEquals(1L, CounterContext.instance().total(row.getCell(cOne).value())); + assertEquals(-1L, CounterContext.instance().total(row.getCell(cTwo).value())); // Remove the first counter, increment the second counter - cells = ArrayBackedSortedColumns.factory.create(cfs.metadata); - cells.addTombstone(cellname(1), (int) System.currentTimeMillis() / 1000, FBUtilities.timestampMicros()); - cells.addCounter(cellname(2), 1L); - new CounterMutation(new Mutation(KEYSPACE1, bytes(1), cells), ConsistencyLevel.ONE).apply(); - current = cfs.getColumnFamily(QueryFilter.getIdentityFilter(dk(bytes(1)), CF1, System.currentTimeMillis())); - assertNull(current.getColumn(cellname(1))); - assertEquals(2L, CounterContext.instance().total(current.getColumn(cellname(2)).value())); + new CounterMutation( + new RowUpdateBuilder(cfs.metadata, 5, "key1") + .clustering("cc") + .delete(cOne) + .add("val2", -5L) + .build(), + ConsistencyLevel.ONE).apply(); + + row = Util.getOnlyRow(Util.cmd(cfs).includeRow("cc").columns("val", "val2").build()); + assertEquals(null, row.getCell(cOne)); + assertEquals(-6L, CounterContext.instance().total(row.getCell(cTwo).value())); // Increment the first counter, make sure it's still shadowed by the tombstone - cells = ArrayBackedSortedColumns.factory.create(cfs.metadata); - cells.addCounter(cellname(1), 1L); - new CounterMutation(new Mutation(KEYSPACE1, bytes(1), cells), ConsistencyLevel.ONE).apply(); - current = cfs.getColumnFamily(QueryFilter.getIdentityFilter(dk(bytes(1)), CF1, System.currentTimeMillis())); - assertNull(current.getColumn(cellname(1))); + new CounterMutation( + new RowUpdateBuilder(cfs.metadata, 5, "key1") + .clustering("cc") + .add("val", 1L) + .build(), + ConsistencyLevel.ONE).apply(); + row = Util.getOnlyRow(Util.cmd(cfs).includeRow("cc").columns("val", "val2").build()); + assertEquals(null, row.getCell(cOne)); // Get rid of the complete partition - Mutation mutation = new Mutation(KEYSPACE1, bytes(1)); - mutation.delete(CF1, FBUtilities.timestampMicros()); - new CounterMutation(mutation, ConsistencyLevel.ONE).apply(); - current = cfs.getColumnFamily(QueryFilter.getIdentityFilter(dk(bytes(1)), CF1, System.currentTimeMillis())); - assertNull(current.getColumn(cellname(1))); - assertNull(current.getColumn(cellname(2))); + RowUpdateBuilder.deleteRow(cfs.metadata, 6, "key1", "cc").applyUnsafe(); + Util.assertEmpty(Util.cmd(cfs).includeRow("cc").columns("val", "val2").build()); // Increment both counters, ensure that both stay dead - cells = ArrayBackedSortedColumns.factory.create(cfs.metadata); - cells.addCounter(cellname(1), 1L); - cells.addCounter(cellname(2), 1L); - new CounterMutation(new Mutation(KEYSPACE1, bytes(1), cells), ConsistencyLevel.ONE).apply(); - current = cfs.getColumnFamily(QueryFilter.getIdentityFilter(dk(bytes(1)), CF1, System.currentTimeMillis())); - assertNull(current.getColumn(cellname(1))); - assertNull(current.getColumn(cellname(2))); - } - - @Test - public void testDuplicateCells() throws WriteTimeoutException - { - ColumnFamilyStore cfs = Keyspace.open(KEYSPACE1).getColumnFamilyStore(CF1); - cfs.truncateBlocking(); - - ColumnFamily cells = ArrayBackedSortedColumns.factory.create(cfs.metadata); - cells.addCounter(cellname(1), 1L); - cells.addCounter(cellname(1), 2L); - cells.addCounter(cellname(1), 3L); - cells.addCounter(cellname(1), 4L); - new CounterMutation(new Mutation(KEYSPACE1, bytes(1), cells), ConsistencyLevel.ONE).apply(); - - ColumnFamily current = cfs.getColumnFamily(QueryFilter.getIdentityFilter(dk(bytes(1)), CF1, System.currentTimeMillis())); - ByteBuffer context = current.getColumn(cellname(1)).value(); - assertEquals(10L, CounterContext.instance().total(context)); - assertEquals(ClockAndCount.create(1L, 10L), CounterContext.instance().getLocalClockAndCount(context)); - assertEquals(ClockAndCount.create(1L, 10L), cfs.getCachedCounter(bytes(1), cellname(1))); + new CounterMutation( + new RowUpdateBuilder(cfs.metadata, 6, "key1") + .clustering("cc") + .add("val", 1L) + .add("val2", 1L) + .build(), + ConsistencyLevel.ONE).apply(); + Util.assertEmpty(Util.cmd(cfs).includeRow("cc").columns("val", "val2").build()); } } diff --git a/test/unit/org/apache/cassandra/db/DeletePartitionTest.java b/test/unit/org/apache/cassandra/db/DeletePartitionTest.java new file mode 100644 index 0000000000..616103e33b --- /dev/null +++ b/test/unit/org/apache/cassandra/db/DeletePartitionTest.java @@ -0,0 +1,97 @@ +/* +* Licensed to the Apache Software Foundation (ASF) under one +* or more contributor license agreements. See the NOTICE file +* distributed with this work for additional information +* regarding copyright ownership. The ASF licenses this file +* to you under the Apache License, Version 2.0 (the +* "License"); you may not use this file except in compliance +* with the License. You may obtain a copy of the License at +* +* http://www.apache.org/licenses/LICENSE-2.0 +* +* Unless required by applicable law or agreed to in writing, +* software distributed under the License is distributed on an +* "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +* KIND, either express or implied. See the License for the +* specific language governing permissions and limitations +* under the License. +*/ +package org.apache.cassandra.db; + +import org.junit.BeforeClass; +import org.junit.Test; + +import org.apache.cassandra.SchemaLoader; +import org.apache.cassandra.Util; +import org.apache.cassandra.config.ColumnDefinition; +import org.apache.cassandra.config.KSMetaData; +import org.apache.cassandra.db.rows.UnfilteredRowIterator; +import org.apache.cassandra.db.rows.Row; +import org.apache.cassandra.db.rows.RowIterator; +import org.apache.cassandra.db.partitions.*; +import org.apache.cassandra.locator.SimpleStrategy; +import org.apache.cassandra.utils.ByteBufferUtil; +import org.apache.cassandra.utils.FBUtilities; + +import static junit.framework.Assert.assertFalse; +import static junit.framework.Assert.assertTrue; + +public class DeletePartitionTest +{ + private static final String KEYSPACE1 = "RemoveColumnFamilyTest"; + private static final String CF_STANDARD1 = "Standard1"; + + @BeforeClass + public static void defineSchema() + { + SchemaLoader.prepareServer(); + SchemaLoader.createKeyspace(KEYSPACE1, + SimpleStrategy.class, + KSMetaData.optsWithRF(1), + SchemaLoader.standardCFMD(KEYSPACE1, CF_STANDARD1)); + } + + @Test + public void testDeletePartition() + { + testDeletePartition(Util.dk("key1"), true, true); + testDeletePartition(Util.dk("key2"), true, false); + testDeletePartition(Util.dk("key3"), false, true); + testDeletePartition(Util.dk("key4"), false, false); + } + + public void testDeletePartition(DecoratedKey key, boolean flushBeforeRemove, boolean flushAfterRemove) + { + ColumnFamilyStore store = Keyspace.open(KEYSPACE1).getColumnFamilyStore(CF_STANDARD1); + ColumnDefinition column = store.metadata.getColumnDefinition(ByteBufferUtil.bytes("val")); + + // write + new RowUpdateBuilder(store.metadata, 0, key.getKey()) + .clustering("Column1") + .add("val", "asdf") + .build() + .applyUnsafe(); + + // validate that data's written + FilteredPartition partition = Util.getOnlyPartition(Util.cmd(store, key).build()); + assertTrue(partition.rowCount() > 0); + Row r = partition.iterator().next(); + assertTrue(r.getCell(column).value().equals(ByteBufferUtil.bytes("asdf"))); + + if (flushBeforeRemove) + store.forceBlockingFlush(); + + // delete the partition + new Mutation(KEYSPACE1, key) + .add(PartitionUpdate.fullPartitionDelete(store.metadata, key, 0, FBUtilities.nowInSeconds())) + .applyUnsafe(); + + if (flushAfterRemove) + store.forceBlockingFlush(); + + // validate removal + ArrayBackedPartition partitionUnfiltered = Util.getOnlyPartitionUnfiltered(Util.cmd(store, key).build()); + assertFalse(partitionUnfiltered.partitionLevelDeletion().isLive()); + assertFalse(partitionUnfiltered.iterator().hasNext()); + } +} diff --git a/test/unit/org/apache/cassandra/db/DirectoriesTest.java b/test/unit/org/apache/cassandra/db/DirectoriesTest.java index 080f01b625..6bb7feeb4b 100644 --- a/test/unit/org/apache/cassandra/db/DirectoriesTest.java +++ b/test/unit/org/apache/cassandra/db/DirectoriesTest.java @@ -19,18 +19,12 @@ package org.apache.cassandra.db; import java.io.File; import java.io.IOException; -import java.util.ArrayList; -import java.util.Arrays; -import java.util.HashMap; -import java.util.HashSet; -import java.util.IdentityHashMap; -import java.util.List; -import java.util.Map; -import java.util.Set; +import java.util.*; import java.util.concurrent.Callable; import java.util.concurrent.Executors; import java.util.concurrent.Future; +import com.google.common.collect.Lists; import org.apache.commons.lang3.StringUtils; import org.junit.AfterClass; @@ -38,36 +32,39 @@ import org.junit.BeforeClass; import org.junit.Test; import org.apache.cassandra.config.CFMetaData; +import org.apache.cassandra.config.ColumnDefinition; import org.apache.cassandra.config.Config.DiskFailurePolicy; import org.apache.cassandra.config.DatabaseDescriptor; import org.apache.cassandra.db.Directories.DataDirectory; +import org.apache.cassandra.db.marshal.UTF8Type; import org.apache.cassandra.io.sstable.Component; import org.apache.cassandra.io.sstable.Descriptor; import org.apache.cassandra.io.util.FileUtils; import org.apache.cassandra.utils.ByteBufferUtil; import org.apache.cassandra.io.FSWriteError; -import static org.junit.Assert.assertEquals; -import static org.junit.Assert.assertFalse; -import static org.junit.Assert.assertSame; -import static org.junit.Assert.assertTrue; -import static org.junit.Assert.fail; +import static org.junit.Assert.*; public class DirectoriesTest { private static File tempDataDir; private static final String KS = "ks"; - private static final String[] CFS = new String[] { "cf1", "ks" }; + private static final String[] TABLES = new String[] { "cf1", "ks" }; - private static final Set CFM = new HashSet<>(CFS.length); + private static final Set CFM = new HashSet<>(TABLES.length); private static Map> files = new HashMap>(); @BeforeClass public static void beforeClass() throws IOException { - for (String cf : CFS) + for (String table : TABLES) { - CFM.add(new CFMetaData(KS, cf, ColumnFamilyType.Standard, null)); + UUID tableID = CFMetaData.generateLegacyCfId(KS, table); + CFM.add(CFMetaData.Builder.create(KS, table) + .withId(tableID) + .addPartitionKey("thekey", UTF8Type.instance) + .addClusteringColumn("thecolumn", UTF8Type.instance) + .build()); } tempDataDir = File.createTempFile("cassandra", "unittest"); @@ -193,8 +190,8 @@ public class DirectoriesTest public void testDiskFailurePolicy_best_effort() { DiskFailurePolicy origPolicy = DatabaseDescriptor.getDiskFailurePolicy(); - - try + + try { DatabaseDescriptor.setDiskFailurePolicy(DiskFailurePolicy.best_effort); // Fake a Directory creation failure diff --git a/test/unit/org/apache/cassandra/db/HintedHandOffTest.java b/test/unit/org/apache/cassandra/db/HintedHandOffTest.java index ce14c3789f..962fc92754 100644 --- a/test/unit/org/apache/cassandra/db/HintedHandOffTest.java +++ b/test/unit/org/apache/cassandra/db/HintedHandOffTest.java @@ -1,5 +1,5 @@ /* - * + * * Licensed to the Apache Software Foundation (ASF) under one * or more contributor license agreements. See the NOTICE file * distributed with this work for additional information @@ -7,16 +7,16 @@ * to you under the Apache License, Version 2.0 (the * "License"); you may not use this file except in compliance * with the License. You may obtain a copy of the License at - * + * * http://www.apache.org/licenses/LICENSE-2.0 - * + * * Unless required by applicable law or agreed to in writing, * software distributed under the License is distributed on an * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY * KIND, either express or implied. See the License for the * specific language governing permissions and limitations * under the License. - * + * */ package org.apache.cassandra.db; @@ -24,13 +24,12 @@ import java.net.InetAddress; import java.util.Map; import java.util.UUID; +import com.google.common.collect.Iterators; +import org.junit.Before; import org.junit.BeforeClass; import org.junit.Test; -import com.google.common.collect.Iterators; - import org.apache.cassandra.SchemaLoader; -import org.apache.cassandra.Util; import org.apache.cassandra.config.KSMetaData; import org.apache.cassandra.cql3.UntypedResultSet; import org.apache.cassandra.db.compaction.SizeTieredCompactionStrategy; @@ -39,16 +38,16 @@ import org.apache.cassandra.db.marshal.UUIDType; import org.apache.cassandra.exceptions.ConfigurationException; import org.apache.cassandra.locator.SimpleStrategy; import org.apache.cassandra.utils.ByteBufferUtil; +import org.apache.cassandra.utils.FBUtilities; -import static org.junit.Assert.assertEquals; import static org.apache.cassandra.cql3.QueryProcessor.executeInternal; +import static org.junit.Assert.assertEquals; public class HintedHandOffTest { public static final String KEYSPACE4 = "HintedHandOffTest4"; public static final String STANDARD1_CF = "Standard1"; - public static final String COLUMN1 = "column1"; @BeforeClass public static void defineSchema() throws ConfigurationException @@ -60,6 +59,14 @@ public class HintedHandOffTest SchemaLoader.standardCFMD(KEYSPACE4, STANDARD1_CF)); } + @Before + public void clearHints() + { + Keyspace systemKeyspace = Keyspace.open("system"); + ColumnFamilyStore hintStore = systemKeyspace.getColumnFamilyStore(SystemKeyspace.HINTS); + hintStore.clearUnsafe(); + } + // Test compaction of hints column family. It shouldn't remove all columns on compaction. @Test public void testCompactionOfHintsCF() throws Exception @@ -73,9 +80,7 @@ public class HintedHandOffTest hintStore.disableAutoCompaction(); // insert 1 hint - Mutation rm = new Mutation(KEYSPACE4, ByteBufferUtil.bytes(1)); - rm.add(STANDARD1_CF, Util.cellname(COLUMN1), ByteBufferUtil.EMPTY_BYTE_BUFFER, System.currentTimeMillis()); - + Mutation rm = mutation(); HintedHandOffManager.instance.hintFor(rm, System.currentTimeMillis(), HintedHandOffManager.calculateHintTTL(rm), @@ -109,14 +114,8 @@ public class HintedHandOffTest @Test(timeout = 5000) public void testTruncateHints() throws Exception { - Keyspace systemKeyspace = Keyspace.open("system"); - ColumnFamilyStore hintStore = systemKeyspace.getColumnFamilyStore(SystemKeyspace.HINTS); - hintStore.clearUnsafe(); - // insert 1 hint - Mutation rm = new Mutation(KEYSPACE4, ByteBufferUtil.bytes(1)); - rm.add(STANDARD1_CF, Util.cellname(COLUMN1), ByteBufferUtil.EMPTY_BYTE_BUFFER, System.currentTimeMillis()); - + Mutation rm = mutation(); HintedHandOffManager.instance.hintFor(rm, System.currentTimeMillis(), HintedHandOffManager.calculateHintTTL(rm), @@ -135,6 +134,17 @@ public class HintedHandOffTest assert getNoOfHints() == 0; } + private Mutation mutation() + { + // get a random mutation to write a hint for + return new RowUpdateBuilder(Keyspace.open(KEYSPACE4).getColumnFamilyStore(STANDARD1_CF).metadata, + FBUtilities.timestampMicros(), + ByteBufferUtil.bytes(1)) + .clustering("cluster_col0") + .add("val", "value0") + .build(); + } + private int getNoOfHints() { String req = "SELECT * FROM system.%s"; diff --git a/test/unit/org/apache/cassandra/db/KeyCacheTest.java b/test/unit/org/apache/cassandra/db/KeyCacheTest.java index 27e7a2b705..76de839349 100644 --- a/test/unit/org/apache/cassandra/db/KeyCacheTest.java +++ b/test/unit/org/apache/cassandra/db/KeyCacheTest.java @@ -32,19 +32,20 @@ import org.junit.Test; import org.apache.cassandra.SchemaLoader; import org.apache.cassandra.Util; import org.apache.cassandra.cache.KeyCacheKey; -import org.apache.cassandra.config.DatabaseDescriptor; -import org.apache.cassandra.config.KSMetaData; import org.apache.cassandra.concurrent.ScheduledExecutors; -import org.apache.cassandra.db.composites.*; +import org.apache.cassandra.config.KSMetaData; +import org.apache.cassandra.config.CFMetaData; +import org.apache.cassandra.config.Schema; import org.apache.cassandra.db.compaction.CompactionManager; -import org.apache.cassandra.db.filter.QueryFilter; +import org.apache.cassandra.db.filter.*; import org.apache.cassandra.exceptions.ConfigurationException; import org.apache.cassandra.io.sstable.format.SSTableReader; import org.apache.cassandra.locator.SimpleStrategy; import org.apache.cassandra.service.CacheService; import org.apache.cassandra.utils.ByteBufferUtil; - +import org.apache.cassandra.utils.FBUtilities; import org.apache.cassandra.utils.concurrent.Refs; + import static org.junit.Assert.assertEquals; public class KeyCacheTest @@ -58,10 +59,10 @@ public class KeyCacheTest { SchemaLoader.prepareServer(); SchemaLoader.createKeyspace(KEYSPACE1, - SimpleStrategy.class, - KSMetaData.optsWithRF(1), - SchemaLoader.standardCFMD(KEYSPACE1, COLUMN_FAMILY1), - SchemaLoader.standardCFMD(KEYSPACE1, COLUMN_FAMILY2)); + SimpleStrategy.class, + KSMetaData.optsWithRF(1), + SchemaLoader.standardCFMD(KEYSPACE1, COLUMN_FAMILY1), + SchemaLoader.standardCFMD(KEYSPACE1, COLUMN_FAMILY2)); } @AfterClass @@ -86,7 +87,7 @@ public class KeyCacheTest store.forceBlockingFlush(); // populate the cache - SchemaLoader.readData(KEYSPACE1, COLUMN_FAMILY2, 0, 100); + readData(KEYSPACE1, COLUMN_FAMILY2, 100); assertKeyCacheSize(100, KEYSPACE1, COLUMN_FAMILY2); // really? our caches don't implement the map interface? (hence no .addAll) @@ -136,37 +137,18 @@ public class KeyCacheTest // KeyCache should start at size 0 if we're caching X% of zero data. assertKeyCacheSize(0, KEYSPACE1, COLUMN_FAMILY1); - DecoratedKey key1 = Util.dk("key1"); - DecoratedKey key2 = Util.dk("key2"); Mutation rm; // inserts - rm = new Mutation(KEYSPACE1, key1.getKey()); - rm.add(COLUMN_FAMILY1, Util.cellname("1"), ByteBufferUtil.EMPTY_BYTE_BUFFER, 0); - rm.applyUnsafe(); - rm = new Mutation(KEYSPACE1, key2.getKey()); - rm.add(COLUMN_FAMILY1, Util.cellname("2"), ByteBufferUtil.EMPTY_BYTE_BUFFER, 0); - rm.applyUnsafe(); + new RowUpdateBuilder(cfs.metadata, 0, "key1").clustering("1").build().applyUnsafe(); + new RowUpdateBuilder(cfs.metadata, 0, "key2").clustering("2").build().applyUnsafe(); // to make sure we have SSTable cfs.forceBlockingFlush(); // reads to cache key position - cfs.getColumnFamily(QueryFilter.getSliceFilter(key1, - COLUMN_FAMILY1, - Composites.EMPTY, - Composites.EMPTY, - false, - 10, - System.currentTimeMillis())); - - cfs.getColumnFamily(QueryFilter.getSliceFilter(key2, - COLUMN_FAMILY1, - Composites.EMPTY, - Composites.EMPTY, - false, - 10, - System.currentTimeMillis())); + Util.getAll(Util.cmd(cfs, "key1").build()); + Util.getAll(Util.cmd(cfs, "key2").build()); assertKeyCacheSize(2, KEYSPACE1, COLUMN_FAMILY1); @@ -176,41 +158,38 @@ public class KeyCacheTest throw new IllegalStateException(); Util.compactAll(cfs, Integer.MAX_VALUE).get(); - boolean noEarlyOpen = DatabaseDescriptor.getSSTablePreempiveOpenIntervalInMB() < 0; - // after compaction cache should have entries for new SSTables, // but since we have kept a reference to the old sstables, // if we had 2 keys in cache previously it should become 4 - assertKeyCacheSize(noEarlyOpen ? 2 : 4, KEYSPACE1, COLUMN_FAMILY1); + assertKeyCacheSize(4, KEYSPACE1, COLUMN_FAMILY1); refs.release(); - Uninterruptibles.sleepUninterruptibly(10, TimeUnit.MILLISECONDS);; - while (ScheduledExecutors.nonPeriodicTasks.getActiveCount() + ScheduledExecutors.nonPeriodicTasks.getQueue().size() > 0); + while (ScheduledExecutors.nonPeriodicTasks.getActiveCount() + ScheduledExecutors.nonPeriodicTasks.getQueue().size() > 0) + { + Uninterruptibles.sleepUninterruptibly(100, TimeUnit.MILLISECONDS);; + } // after releasing the reference this should drop to 2 assertKeyCacheSize(2, KEYSPACE1, COLUMN_FAMILY1); // re-read same keys to verify that key cache didn't grow further - cfs.getColumnFamily(QueryFilter.getSliceFilter(key1, - COLUMN_FAMILY1, - Composites.EMPTY, - Composites.EMPTY, - false, - 10, - System.currentTimeMillis())); + Util.getAll(Util.cmd(cfs, "key1").build()); + Util.getAll(Util.cmd(cfs, "key2").build()); - cfs.getColumnFamily(QueryFilter.getSliceFilter(key2, - COLUMN_FAMILY1, - Composites.EMPTY, - Composites.EMPTY, - false, - 10, - System.currentTimeMillis())); - - assertKeyCacheSize(noEarlyOpen ? 4 : 2, KEYSPACE1, COLUMN_FAMILY1); + assertKeyCacheSize(2, KEYSPACE1, COLUMN_FAMILY1); } + private static void readData(String keyspace, String columnFamily, int numberOfRows) + { + ColumnFamilyStore store = Keyspace.open(keyspace).getColumnFamilyStore(columnFamily); + CFMetaData cfm = Schema.instance.getCFMetaData(keyspace, columnFamily); + + for (int i = 0; i < numberOfRows; i++) + Util.getAll(Util.cmd(store, "key" + i).includeRow("col" + i).build()); + } + + private void assertKeyCacheSize(int expected, String keyspace, String columnFamily) { int size = 0; diff --git a/test/unit/org/apache/cassandra/db/KeyspaceTest.java b/test/unit/org/apache/cassandra/db/KeyspaceTest.java index d9481caeaf..fdd4f0c565 100644 --- a/test/unit/org/apache/cassandra/db/KeyspaceTest.java +++ b/test/unit/org/apache/cassandra/db/KeyspaceTest.java @@ -19,639 +19,492 @@ package org.apache.cassandra.db; import java.nio.ByteBuffer; -import java.nio.charset.CharacterCodingException; -import java.text.DecimalFormat; -import java.text.NumberFormat; import java.util.*; -import java.io.IOException; -import com.google.common.collect.Iterables; +import org.apache.cassandra.Util; +import org.apache.cassandra.cql3.CQLTester; +import org.apache.cassandra.cql3.ColumnIdentifier; +import org.apache.cassandra.db.rows.Cell; +import org.apache.cassandra.db.rows.Row; +import org.apache.cassandra.db.rows.RowIterator; +import org.apache.cassandra.db.compaction.CompactionManager; +import org.apache.cassandra.db.filter.*; +import org.apache.cassandra.db.partitions.PartitionIterator; import org.apache.cassandra.io.sstable.format.SSTableReader; -import org.apache.commons.lang3.StringUtils; -import org.junit.BeforeClass; +import org.apache.cassandra.metrics.ClearableHistogram; +import org.apache.cassandra.utils.ByteBufferUtil; +import org.apache.cassandra.utils.FBUtilities; import org.junit.Test; import static org.junit.Assert.*; -import org.apache.cassandra.SchemaLoader; -import org.apache.cassandra.config.CFMetaData; -import org.apache.cassandra.config.DatabaseDescriptor; -import org.apache.cassandra.config.KSMetaData; -import org.apache.cassandra.db.composites.*; -import org.apache.cassandra.db.compaction.CompactionManager; -import org.apache.cassandra.db.filter.QueryFilter; -import org.apache.cassandra.db.marshal.AbstractType; -import org.apache.cassandra.db.marshal.BytesType; -import org.apache.cassandra.db.marshal.CompositeType; -import org.apache.cassandra.db.marshal.IntegerType; -import org.apache.cassandra.exceptions.ConfigurationException; -import org.apache.cassandra.locator.SimpleStrategy; -import org.apache.cassandra.metrics.ClearableHistogram; -import org.apache.cassandra.utils.WrappedRunnable; -import static org.apache.cassandra.Util.column; -import static org.apache.cassandra.Util.expiringColumn; -import static org.apache.cassandra.Util.cellname; -import org.apache.cassandra.Util; -import org.apache.cassandra.utils.ByteBufferUtil; -public class KeyspaceTest +public class KeyspaceTest extends CQLTester { - private static final DecoratedKey TEST_KEY = Util.dk("key1"); - private static final DecoratedKey TEST_SLICE_KEY = Util.dk("key1-slicerange"); - - private static final String KEYSPACE1 = "Keyspace1"; - private static final String CF_STANDARD1 = "Standard1"; - private static final String CF_STANDARD2 = "Standard2"; - private static final String CF_STANDARDLONG = "StandardLong1"; - private static final String CF_STANDARDCOMPOSITE2 = "StandardComposite2"; - - private static final String KEYSPACE2 = "Keyspace2"; - private static final String CF_STANDARD3 = "Standard3"; - - @BeforeClass - public static void defineSchema() throws ConfigurationException - { - SchemaLoader.prepareServer(); - AbstractType compositeMaxMin = CompositeType.getInstance(Arrays.asList(new AbstractType[]{BytesType.instance, IntegerType.instance})); - SchemaLoader.createKeyspace(KEYSPACE1, - SimpleStrategy.class, - KSMetaData.optsWithRF(1), - SchemaLoader.standardCFMD(KEYSPACE1, CF_STANDARD1), - SchemaLoader.standardCFMD(KEYSPACE1, CF_STANDARD2), - SchemaLoader.standardCFMD(KEYSPACE1, CF_STANDARDLONG), - SchemaLoader.standardCFMD(KEYSPACE1, CF_STANDARDCOMPOSITE2, compositeMaxMin)); - SchemaLoader.createKeyspace(KEYSPACE2, - SimpleStrategy.class, - KSMetaData.optsWithRF(1), - SchemaLoader.standardCFMD(KEYSPACE2, CF_STANDARD3)); - } - - public static void reTest(ColumnFamilyStore cfs, Runnable verify) throws Exception - { - verify.run(); - cfs.forceBlockingFlush(); - verify.run(); - } - @Test public void testGetRowNoColumns() throws Throwable { - final Keyspace keyspace = Keyspace.open(KEYSPACE2); - final ColumnFamilyStore cfStore = keyspace.getColumnFamilyStore("Standard3"); + String tableName = createTable("CREATE TABLE %s (a text, b int, c int, PRIMARY KEY (a, b))"); - ColumnFamily cf = ArrayBackedSortedColumns.factory.create(KEYSPACE2, "Standard3"); - cf.addColumn(column("col1","val1", 1L)); - Mutation rm = new Mutation(KEYSPACE2, TEST_KEY.getKey(), cf); - rm.applyUnsafe(); + execute("INSERT INTO %s (a, b, c) VALUES (?, ?, ?)", "0", 0, 0); - Runnable verify = new WrappedRunnable() + final ColumnFamilyStore cfs = Keyspace.open(KEYSPACE).getColumnFamilyStore(tableName); + + for (int round = 0; round < 2; round++) { - public void runMayThrow() throws Exception - { - ColumnFamily cf; + // slice with limit 0 + Util.assertEmpty(Util.cmd(cfs, "0").columns("c").withLimit(0).build()); - cf = cfStore.getColumnFamily(Util.namesQueryFilter(cfStore, TEST_KEY)); - assertColumns(cf); + // slice with nothing in between the bounds + Util.assertEmpty(Util.cmd(cfs, "0").columns("c").fromIncl(1).toIncl(1).build()); - cf = cfStore.getColumnFamily(QueryFilter.getSliceFilter(TEST_KEY, "Standard3", Composites.EMPTY, Composites.EMPTY, false, 0, System.currentTimeMillis())); - assertColumns(cf); + // fetch a non-existent name + Util.assertEmpty(Util.cmd(cfs, "0").columns("c").includeRow(1).build()); - cf = cfStore.getColumnFamily(Util.namesQueryFilter(cfStore, TEST_KEY, "col99")); - assertColumns(cf); - } - }; - reTest(keyspace.getColumnFamilyStore("Standard3"), verify); + if (round == 0) + cfs.forceBlockingFlush(); + } } @Test public void testGetRowSingleColumn() throws Throwable { - final Keyspace keyspace = Keyspace.open(KEYSPACE1); - final ColumnFamilyStore cfStore = keyspace.getColumnFamilyStore("Standard1"); + String tableName = createTable("CREATE TABLE %s (a text, b int, c int, PRIMARY KEY (a, b))"); - ColumnFamily cf = ArrayBackedSortedColumns.factory.create(KEYSPACE1, "Standard1"); - cf.addColumn(column("col1","val1", 1L)); - cf.addColumn(column("col2","val2", 1L)); - cf.addColumn(column("col3","val3", 1L)); - Mutation rm = new Mutation(KEYSPACE1, TEST_KEY.getKey(), cf); - rm.applyUnsafe(); + for (int i = 0; i < 2; i++) + execute("INSERT INTO %s (a, b, c) VALUES (?, ?, ?)", "0", i, i); - Runnable verify = new WrappedRunnable() + final ColumnFamilyStore cfs = Keyspace.open(KEYSPACE).getColumnFamilyStore(tableName); + + for (int round = 0; round < 2; round++) { - public void runMayThrow() throws Exception + // slice with limit 1 + Row row = Util.getOnlyRow(Util.cmd(cfs, "0").columns("c").withLimit(1).build()); + assertEquals(ByteBufferUtil.bytes(0), row.getCell(cfs.metadata.getColumnDefinition(new ColumnIdentifier("c", false))).value()); + + // fetch each row by name + for (int i = 0; i < 2; i++) { - ColumnFamily cf; - - cf = cfStore.getColumnFamily(Util.namesQueryFilter(cfStore, TEST_KEY, "col1")); - assertColumns(cf, "col1"); - - cf = cfStore.getColumnFamily(Util.namesQueryFilter(cfStore, TEST_KEY, "col3")); - assertColumns(cf, "col3"); + row = Util.getOnlyRow(Util.cmd(cfs, "0").columns("c").includeRow(i).build()); + assertEquals(ByteBufferUtil.bytes(i), row.getCell(cfs.metadata.getColumnDefinition(new ColumnIdentifier("c", false))).value()); } - }; - reTest(keyspace.getColumnFamilyStore("Standard1"), verify); + + // fetch each row by slice + for (int i = 0; i < 2; i++) + { + row = Util.getOnlyRow(Util.cmd(cfs, "0").columns("c").fromIncl(i).toIncl(i).build()); + assertEquals(ByteBufferUtil.bytes(i), row.getCell(cfs.metadata.getColumnDefinition(new ColumnIdentifier("c", false))).value()); + } + + if (round == 0) + cfs.forceBlockingFlush(); + } } @Test - public void testGetRowSliceByRange() throws Throwable + public void testGetSliceBloomFilterFalsePositive() throws Throwable { - DecoratedKey key = TEST_SLICE_KEY; - Keyspace keyspace = Keyspace.open(KEYSPACE1); - ColumnFamilyStore cfStore = keyspace.getColumnFamilyStore("Standard1"); - ColumnFamily cf = ArrayBackedSortedColumns.factory.create(KEYSPACE1, "Standard1"); - // First write "a", "b", "c" - cf.addColumn(column("a", "val1", 1L)); - cf.addColumn(column("b", "val2", 1L)); - cf.addColumn(column("c", "val3", 1L)); - Mutation rm = new Mutation(KEYSPACE1, key.getKey(), cf); - rm.applyUnsafe(); + String tableName = createTable("CREATE TABLE %s (a text, b int, c int, PRIMARY KEY (a, b))"); - cf = cfStore.getColumnFamily(key, cellname("b"), cellname("c"), false, 100, System.currentTimeMillis()); - assertEquals(2, cf.getColumnCount()); + execute("INSERT INTO %s (a, b, c) VALUES (?, ?, ?)", "1", 1, 1); - cf = cfStore.getColumnFamily(key, cellname("b"), cellname("b"), false, 100, System.currentTimeMillis()); - assertEquals(1, cf.getColumnCount()); + final ColumnFamilyStore cfs = Keyspace.open(KEYSPACE).getColumnFamilyStore(tableName); - cf = cfStore.getColumnFamily(key, cellname("b"), cellname("c"), false, 1, System.currentTimeMillis()); - assertEquals(1, cf.getColumnCount()); + // check empty reads on the partitions before and after the existing one + for (String key : new String[]{"0", "2"}) + Util.assertEmpty(Util.cmd(cfs, key).build()); + + cfs.forceBlockingFlush(); + + for (String key : new String[]{"0", "2"}) + Util.assertEmpty(Util.cmd(cfs, key).build()); + + Collection sstables = cfs.getSSTables(); + assertEquals(1, sstables.size()); + sstables.iterator().next().forceFilterFailures(); + + for (String key : new String[]{"0", "2"}) + Util.assertEmpty(Util.cmd(cfs, key).build()); } - @Test - public void testGetSliceNoMatch() throws Throwable + private static void assertRowsInSlice(ColumnFamilyStore cfs, String key, int sliceStart, int sliceEnd, int limit, boolean reversed, String columnValuePrefix) { - Keyspace keyspace = Keyspace.open(KEYSPACE1); - ColumnFamily cf = ArrayBackedSortedColumns.factory.create(KEYSPACE1, "Standard2"); - cf.addColumn(column("col1", "val1", 1)); - Mutation rm = new Mutation(KEYSPACE1, ByteBufferUtil.bytes("row1000"), cf); - rm.applyUnsafe(); + Clustering startClustering = new SimpleClustering(ByteBufferUtil.bytes(sliceStart)); + Clustering endClustering = new SimpleClustering(ByteBufferUtil.bytes(sliceEnd)); + Slices slices = Slices.with(cfs.getComparator(), Slice.make(startClustering, endClustering)); + ClusteringIndexSliceFilter filter = new ClusteringIndexSliceFilter(slices, reversed); + SinglePartitionSliceCommand command = singlePartitionSlice(cfs, key, filter, limit); - validateGetSliceNoMatch(keyspace); - keyspace.getColumnFamilyStore("Standard2").forceBlockingFlush(); - validateGetSliceNoMatch(keyspace); - - Collection ssTables = keyspace.getColumnFamilyStore("Standard2").getSSTables(); - assertEquals(1, ssTables.size()); - ssTables.iterator().next().forceFilterFailures(); - validateGetSliceNoMatch(keyspace); + try (ReadOrderGroup orderGroup = command.startOrderGroup(); PartitionIterator iterator = command.executeInternal(orderGroup)) + { + try (RowIterator rowIterator = iterator.next()) + { + if (reversed) + { + for (int i = sliceEnd; i >= sliceStart; i--) + { + Row row = rowIterator.next(); + Cell cell = row.getCell(cfs.metadata.getColumnDefinition(new ColumnIdentifier("c", false))); + assertEquals(ByteBufferUtil.bytes(columnValuePrefix + i), cell.value()); + } + } + else + { + for (int i = sliceStart; i <= sliceEnd; i++) + { + Row row = rowIterator.next(); + Cell cell = row.getCell(cfs.metadata.getColumnDefinition(new ColumnIdentifier("c", false))); + assertEquals(ByteBufferUtil.bytes(columnValuePrefix + i), cell.value()); + } + } + assertFalse(rowIterator.hasNext()); + } + } } @Test public void testGetSliceWithCutoff() throws Throwable { // tests slicing against data from one row in a memtable and then flushed to an sstable - final Keyspace keyspace = Keyspace.open(KEYSPACE1); - final ColumnFamilyStore cfStore = keyspace.getColumnFamilyStore("Standard1"); - final DecoratedKey ROW = Util.dk("row4"); - final NumberFormat fmt = new DecimalFormat("000"); + String tableName = createTable("CREATE TABLE %s (a text, b int, c text, PRIMARY KEY (a, b))"); + String prefix = "omg!thisisthevalue!"; - ColumnFamily cf = ArrayBackedSortedColumns.factory.create(KEYSPACE1, "Standard1"); - // at this rate, we're getting 78-79 cos/block, assuming the blocks are set to be about 4k. - // so if we go to 300, we'll get at least 4 blocks, which is plenty for testing. for (int i = 0; i < 300; i++) - cf.addColumn(column("col" + fmt.format(i), "omg!thisisthevalue!"+i, 1L)); - Mutation rm = new Mutation(KEYSPACE1, ROW.getKey(), cf); - rm.applyUnsafe(); + execute("INSERT INTO %s (a, b, c) VALUES (?, ?, ?)", "0", i, prefix + i); - Runnable verify = new WrappedRunnable() + final ColumnFamilyStore cfs = Keyspace.open(KEYSPACE).getColumnFamilyStore(tableName); + + for (int round = 0; round < 2; round++) { - public void runMayThrow() throws Exception - { - ColumnFamily cf; + assertRowsInSlice(cfs, "0", 96, 99, 4, false, prefix); + assertRowsInSlice(cfs, "0", 96, 99, 4, true, prefix); - // blocks are partitioned like this: 000-097, 098-193, 194-289, 290-299, assuming a 4k column index size. - assert DatabaseDescriptor.getColumnIndexSize() == 4096 : "Unexpected column index size, block boundaries won't be where tests expect them."; + assertRowsInSlice(cfs, "0", 100, 103, 4, false, prefix); + assertRowsInSlice(cfs, "0", 100, 103, 4, true, prefix); - // test forward, spanning a segment. - cf = cfStore.getColumnFamily(ROW, cellname("col096"), cellname("col099"), false, 4, System.currentTimeMillis()); - assertColumns(cf, "col096", "col097", "col098", "col099"); + assertRowsInSlice(cfs, "0", 0, 99, 100, false, prefix); + assertRowsInSlice(cfs, "0", 288, 299, 12, true, prefix); - // test reversed, spanning a segment. - cf = cfStore.getColumnFamily(ROW, cellname("col099"), cellname("col096"), true, 4, System.currentTimeMillis()); - assertColumns(cf, "col096", "col097", "col098", "col099"); - - // test forward, within a segment. - cf = cfStore.getColumnFamily(ROW, cellname("col100"), cellname("col103"), false, 4, System.currentTimeMillis()); - assertColumns(cf, "col100", "col101", "col102", "col103"); - - // test reversed, within a segment. - cf = cfStore.getColumnFamily(ROW, cellname("col103"), cellname("col100"), true, 4, System.currentTimeMillis()); - assertColumns(cf, "col100", "col101", "col102", "col103"); - - // test forward from beginning, spanning a segment. - String[] strCols = new String[100]; // col000-col099 - for (int i = 0; i < 100; i++) - strCols[i] = "col" + fmt.format(i); - cf = cfStore.getColumnFamily(ROW, Composites.EMPTY, cellname("col099"), false, 100, System.currentTimeMillis()); - assertColumns(cf, strCols); - - // test reversed, from end, spanning a segment. - cf = cfStore.getColumnFamily(ROW, Composites.EMPTY, cellname("col288"), true, 12, System.currentTimeMillis()); - assertColumns(cf, "col288", "col289", "col290", "col291", "col292", "col293", "col294", "col295", "col296", "col297", "col298", "col299"); - } - }; - - reTest(keyspace.getColumnFamilyStore("Standard1"), verify); + if (round == 0) + cfs.forceBlockingFlush(); + } } @Test - public void testReversedWithFlushing() + public void testReversedWithFlushing() throws Throwable { - final Keyspace keyspace = Keyspace.open(KEYSPACE1); - final ColumnFamilyStore cfs = keyspace.getColumnFamilyStore("StandardLong1"); - final DecoratedKey ROW = Util.dk("row4"); + String tableName = createTable("CREATE TABLE %s (a text, b int, c int, PRIMARY KEY (a, b)) WITH CLUSTERING ORDER BY (b DESC)"); + final ColumnFamilyStore cfs = Keyspace.open(KEYSPACE).getColumnFamilyStore(tableName); for (int i = 0; i < 10; i++) - { - ColumnFamily cf = ArrayBackedSortedColumns.factory.create(KEYSPACE1, "StandardLong1"); - cf.addColumn(new BufferCell(cellname((long)i), ByteBufferUtil.EMPTY_BYTE_BUFFER, 0)); - Mutation rm = new Mutation(KEYSPACE1, ROW.getKey(), cf); - rm.applyUnsafe(); - } + execute("INSERT INTO %s (a, b, c) VALUES (?, ?, ?)", "0", i, i); cfs.forceBlockingFlush(); for (int i = 10; i < 20; i++) { - ColumnFamily cf = ArrayBackedSortedColumns.factory.create(KEYSPACE1, "StandardLong1"); - cf.addColumn(new BufferCell(cellname((long)i), ByteBufferUtil.EMPTY_BYTE_BUFFER, 0)); - Mutation rm = new Mutation(KEYSPACE1, ROW.getKey(), cf); - rm.applyUnsafe(); + execute("INSERT INTO %s (a, b, c) VALUES (?, ?, ?)", "0", i, i); - cf = cfs.getColumnFamily(ROW, Composites.EMPTY, Composites.EMPTY, true, 1, System.currentTimeMillis()); - assertEquals(1, Iterables.size(cf.getColumnNames())); - assertEquals(i, cf.getColumnNames().iterator().next().toByteBuffer().getLong()); + PartitionColumns columns = PartitionColumns.of(cfs.metadata.getColumnDefinition(new ColumnIdentifier("c", false))); + ClusteringIndexSliceFilter filter = new ClusteringIndexSliceFilter(Slices.ALL, false); + SinglePartitionSliceCommand command = singlePartitionSlice(cfs, "0", filter, null); + try (ReadOrderGroup orderGroup = command.startOrderGroup(); PartitionIterator iterator = command.executeInternal(orderGroup)) + { + try (RowIterator rowIterator = iterator.next()) + { + Row row = rowIterator.next(); + Cell cell = row.getCell(cfs.metadata.getColumnDefinition(new ColumnIdentifier("c", false))); + assertEquals(ByteBufferUtil.bytes(i), cell.value()); + } + } } } - private void validateGetSliceNoMatch(Keyspace keyspace) + private static void assertRowsInResult(ColumnFamilyStore cfs, SinglePartitionReadCommand command, int ... columnValues) { - ColumnFamilyStore cfStore = keyspace.getColumnFamilyStore("Standard2"); - ColumnFamily cf; + try (ReadOrderGroup orderGroup = command.startOrderGroup(); PartitionIterator iterator = command.executeInternal(orderGroup)) + { + if (columnValues.length == 0) + { + if (iterator.hasNext()) + fail("Didn't expect any results, but got rows starting with: " + iterator.next().next().toString(cfs.metadata)); + return; + } - // key before the rows that exists - cf = cfStore.getColumnFamily(Util.dk("a"), Composites.EMPTY, Composites.EMPTY, false, 1, System.currentTimeMillis()); - assertColumns(cf); + try (RowIterator rowIterator = iterator.next()) + { + for (int expected : columnValues) + { + Row row = rowIterator.next(); + Cell cell = row.getCell(cfs.metadata.getColumnDefinition(new ColumnIdentifier("c", false))); + assertEquals( + String.format("Expected %s, but got %s", ByteBufferUtil.bytesToHex(ByteBufferUtil.bytes(expected)), ByteBufferUtil.bytesToHex(cell.value())), + ByteBufferUtil.bytes(expected), cell.value()); + } + assertFalse(rowIterator.hasNext()); + } + } + } - // key after the rows that exist - cf = cfStore.getColumnFamily(Util.dk("z"), Composites.EMPTY, Composites.EMPTY, false, 1, System.currentTimeMillis()); - assertColumns(cf); + private static ClusteringIndexSliceFilter slices(ColumnFamilyStore cfs, Integer sliceStart, Integer sliceEnd, boolean reversed) + { + Slice.Bound startBound = sliceStart == null + ? Slice.Bound.BOTTOM + : Slice.Bound.create(ClusteringPrefix.Kind.INCL_START_BOUND, new ByteBuffer[]{ByteBufferUtil.bytes(sliceStart)}); + Slice.Bound endBound = sliceEnd == null + ? Slice.Bound.TOP + : Slice.Bound.create(ClusteringPrefix.Kind.INCL_END_BOUND, new ByteBuffer[]{ByteBufferUtil.bytes(sliceEnd)}); + Slices slices = Slices.with(cfs.getComparator(), Slice.make(startBound, endBound)); + return new ClusteringIndexSliceFilter(slices, reversed); + } + + private static SinglePartitionSliceCommand singlePartitionSlice(ColumnFamilyStore cfs, String key, ClusteringIndexSliceFilter filter, Integer rowLimit) + { + DataLimits limit = rowLimit == null + ? DataLimits.NONE + : DataLimits.cqlLimits(rowLimit); + return new SinglePartitionSliceCommand( + cfs.metadata, FBUtilities.nowInSeconds(), ColumnFilter.all(cfs.metadata), RowFilter.NONE, limit, Util.dk(key), filter); } @Test public void testGetSliceFromBasic() throws Throwable { // tests slicing against data from one row in a memtable and then flushed to an sstable - final Keyspace keyspace = Keyspace.open(KEYSPACE1); - final ColumnFamilyStore cfStore = keyspace.getColumnFamilyStore("Standard1"); - final DecoratedKey ROW = Util.dk("row1"); + String tableName = createTable("CREATE TABLE %s (a text, b int, c int, PRIMARY KEY (a, b))"); + final ColumnFamilyStore cfs = Keyspace.open(KEYSPACE).getColumnFamilyStore(tableName); - ColumnFamily cf = ArrayBackedSortedColumns.factory.create(KEYSPACE1, "Standard1"); - cf.addColumn(column("col1", "val1", 1L)); - cf.addColumn(column("col3", "val3", 1L)); - cf.addColumn(column("col4", "val4", 1L)); - cf.addColumn(column("col5", "val5", 1L)); - cf.addColumn(column("col7", "val7", 1L)); - cf.addColumn(column("col9", "val9", 1L)); - Mutation rm = new Mutation(KEYSPACE1, ROW.getKey(), cf); - rm.applyUnsafe(); - - rm = new Mutation(KEYSPACE1, ROW.getKey()); - rm.delete("Standard1", cellname("col4"), 2L); - rm.applyUnsafe(); - - Runnable verify = new WrappedRunnable() + for (int i = 1; i < 10; i++) { - public void runMayThrow() throws Exception - { - ColumnFamily cf; + if (i == 6 || i == 8) + continue; + execute("INSERT INTO %s (a, b, c) VALUES (?, ?, ?)", "0", i, i); + } - cf = cfStore.getColumnFamily(ROW, cellname("col5"), Composites.EMPTY, false, 2, System.currentTimeMillis()); - assertColumns(cf, "col5", "col7"); + execute("DELETE FROM %s WHERE a = ? AND b = ?", "0", 4); - cf = cfStore.getColumnFamily(ROW, cellname("col4"), Composites.EMPTY, false, 2, System.currentTimeMillis()); - assertColumns(cf, "col4", "col5", "col7"); - assertColumns(ColumnFamilyStore.removeDeleted(cf, Integer.MAX_VALUE), "col5", "col7"); + for (int round = 0; round < 2; round++) + { - cf = cfStore.getColumnFamily(ROW, cellname("col5"), Composites.EMPTY, true, 2, System.currentTimeMillis()); - assertColumns(cf, "col3", "col4", "col5"); + ClusteringIndexSliceFilter filter = slices(cfs, 5, null, false); + SinglePartitionSliceCommand command = singlePartitionSlice(cfs, "0", filter, 2); + assertRowsInResult(cfs, command, 5, 7); - cf = cfStore.getColumnFamily(ROW, cellname("col6"), Composites.EMPTY, true, 2, System.currentTimeMillis()); - assertColumns(cf, "col3", "col4", "col5"); + command = singlePartitionSlice(cfs, "0", slices(cfs, 4, null, false), 2); + assertRowsInResult(cfs, command, 5, 7); - cf = cfStore.getColumnFamily(ROW, Composites.EMPTY, Composites.EMPTY, true, 2, System.currentTimeMillis()); - assertColumns(cf, "col7", "col9"); + command = singlePartitionSlice(cfs, "0", slices(cfs, null, 5, true), 2); + assertRowsInResult(cfs, command, 5, 3); - cf = cfStore.getColumnFamily(ROW, cellname("col95"), Composites.EMPTY, false, 2, System.currentTimeMillis()); - assertColumns(cf); + command = singlePartitionSlice(cfs, "0", slices(cfs, null, 6, true), 2); + assertRowsInResult(cfs, command, 5, 3); - cf = cfStore.getColumnFamily(ROW, cellname("col0"), Composites.EMPTY, true, 2, System.currentTimeMillis()); - assertColumns(cf); - } - }; + command = singlePartitionSlice(cfs, "0", slices(cfs, null, 6, true), 2); + assertRowsInResult(cfs, command, 5, 3); - reTest(keyspace.getColumnFamilyStore("Standard1"), verify); + command = singlePartitionSlice(cfs, "0", slices(cfs, null, null, true), 2); + assertRowsInResult(cfs, command, 9, 7); + + command = singlePartitionSlice(cfs, "0", slices(cfs, 95, null, false), 2); + assertRowsInResult(cfs, command); + + command = singlePartitionSlice(cfs, "0", slices(cfs, null, 0, true), 2); + assertRowsInResult(cfs, command); + + if (round == 0) + cfs.forceBlockingFlush(); + } } @Test public void testGetSliceWithExpiration() throws Throwable { // tests slicing against data from one row with expiring column in a memtable and then flushed to an sstable - final Keyspace keyspace = Keyspace.open(KEYSPACE1); - final ColumnFamilyStore cfStore = keyspace.getColumnFamilyStore("Standard1"); - final DecoratedKey ROW = Util.dk("row5"); + String tableName = createTable("CREATE TABLE %s (a text, b int, c int, PRIMARY KEY (a, b))"); + final ColumnFamilyStore cfs = Keyspace.open(KEYSPACE).getColumnFamilyStore(tableName); - ColumnFamily cf = ArrayBackedSortedColumns.factory.create(KEYSPACE1, "Standard1"); - cf.addColumn(column("col1", "val1", 1L)); - cf.addColumn(expiringColumn("col2", "val2", 1L, 60)); // long enough not to be tombstoned - cf.addColumn(column("col3", "val3", 1L)); - Mutation rm = new Mutation(KEYSPACE1, ROW.getKey(), cf); - rm.applyUnsafe(); + execute("INSERT INTO %s (a, b, c) VALUES (?, ?, ?)", "0", 0, 0); + execute("INSERT INTO %s (a, b, c) VALUES (?, ?, ?) USING TTL 60", "0", 1, 1); + execute("INSERT INTO %s (a, b, c) VALUES (?, ?, ?)", "0", 2, 2); - Runnable verify = new WrappedRunnable() + for (int round = 0; round < 2; round++) { - public void runMayThrow() throws Exception - { - ColumnFamily cf; + SinglePartitionSliceCommand command = singlePartitionSlice(cfs, "0", slices(cfs, null, null, false), 2); + assertRowsInResult(cfs, command, 0, 1); - cf = cfStore.getColumnFamily(ROW, Composites.EMPTY, Composites.EMPTY, false, 2, System.currentTimeMillis()); - assertColumns(cf, "col1", "col2"); - assertColumns(ColumnFamilyStore.removeDeleted(cf, Integer.MAX_VALUE), "col1"); + command = singlePartitionSlice(cfs, "0", slices(cfs, 1, null, false), 1); + assertRowsInResult(cfs, command, 1); - cf = cfStore.getColumnFamily(ROW, cellname("col2"), Composites.EMPTY, false, 1, System.currentTimeMillis()); - assertColumns(cf, "col2"); - assertColumns(ColumnFamilyStore.removeDeleted(cf, Integer.MAX_VALUE)); - } - }; - - reTest(keyspace.getColumnFamilyStore("Standard1"), verify); + if (round == 0) + cfs.forceBlockingFlush(); + } } @Test public void testGetSliceFromAdvanced() throws Throwable { // tests slicing against data from one row spread across two sstables - final Keyspace keyspace = Keyspace.open(KEYSPACE1); - final ColumnFamilyStore cfStore = keyspace.getColumnFamilyStore("Standard1"); - final DecoratedKey ROW = Util.dk("row2"); + String tableName = createTable("CREATE TABLE %s (a text, b int, c int, PRIMARY KEY (a, b))"); + final ColumnFamilyStore cfs = Keyspace.open(KEYSPACE).getColumnFamilyStore(tableName); - ColumnFamily cf = ArrayBackedSortedColumns.factory.create(KEYSPACE1, "Standard1"); - cf.addColumn(column("col1", "val1", 1L)); - cf.addColumn(column("col2", "val2", 1L)); - cf.addColumn(column("col3", "val3", 1L)); - cf.addColumn(column("col4", "val4", 1L)); - cf.addColumn(column("col5", "val5", 1L)); - cf.addColumn(column("col6", "val6", 1L)); - Mutation rm = new Mutation(KEYSPACE1, ROW.getKey(), cf); - rm.applyUnsafe(); - cfStore.forceBlockingFlush(); + for (int i = 1; i < 7; i++) + execute("INSERT INTO %s (a, b, c) VALUES (?, ?, ?)", "0", i, i); - cf = ArrayBackedSortedColumns.factory.create(KEYSPACE1, "Standard1"); - cf.addColumn(column("col1", "valx", 2L)); - cf.addColumn(column("col2", "valx", 2L)); - cf.addColumn(column("col3", "valx", 2L)); - rm = new Mutation(KEYSPACE1, ROW.getKey(), cf); - rm.applyUnsafe(); + cfs.forceBlockingFlush(); - Runnable verify = new WrappedRunnable() + // overwrite three rows with -1 + for (int i = 1; i < 4; i++) + execute("INSERT INTO %s (a, b, c) VALUES (?, ?, ?)", "0", i, -1); + + for (int round = 0; round < 2; round++) { - public void runMayThrow() throws Exception - { - ColumnFamily cf; + SinglePartitionSliceCommand command = singlePartitionSlice(cfs, "0", slices(cfs, 2, null, false), 3); + assertRowsInResult(cfs, command, -1, -1, 4); - cf = cfStore.getColumnFamily(ROW, cellname("col2"), Composites.EMPTY, false, 3, System.currentTimeMillis()); - assertColumns(cf, "col2", "col3", "col4"); - - ByteBuffer col = cf.getColumn(cellname("col2")).value(); - assertEquals(ByteBufferUtil.string(col), "valx"); - - col = cf.getColumn(cellname("col3")).value(); - assertEquals(ByteBufferUtil.string(col), "valx"); - - col = cf.getColumn(cellname("col4")).value(); - assertEquals(ByteBufferUtil.string(col), "val4"); - } - }; - - reTest(keyspace.getColumnFamilyStore("Standard1"), verify); + if (round == 0) + cfs.forceBlockingFlush(); + } } @Test public void testGetSliceFromLarge() throws Throwable { - // tests slicing against 1000 columns in an sstable - Keyspace keyspace = Keyspace.open(KEYSPACE1); - ColumnFamilyStore cfStore = keyspace.getColumnFamilyStore("Standard1"); - DecoratedKey key = Util.dk("row3"); - ColumnFamily cf = ArrayBackedSortedColumns.factory.create(KEYSPACE1, "Standard1"); - for (int i = 1000; i < 2000; i++) - cf.addColumn(column("col" + i, ("v" + i), 1L)); - Mutation rm = new Mutation(KEYSPACE1, key.getKey(), cf); - rm.applyUnsafe(); - cfStore.forceBlockingFlush(); + // tests slicing against 1000 rows in an sstable + String tableName = createTable("CREATE TABLE %s (a text, b int, c int, PRIMARY KEY (a, b))"); + final ColumnFamilyStore cfs = Keyspace.open(KEYSPACE).getColumnFamilyStore(tableName); - validateSliceLarge(cfStore); + for (int i = 1000; i < 2000; i++) + execute("INSERT INTO %s (a, b, c) VALUES (?, ?, ?)", "0", i, i); + + cfs.forceBlockingFlush(); + + validateSliceLarge(cfs); // compact so we have a big row with more than the minimum index count - if (cfStore.getSSTables().size() > 1) - { - CompactionManager.instance.performMaximal(cfStore, false); - } + if (cfs.getSSTables().size() > 1) + CompactionManager.instance.performMaximal(cfs, false); + // verify that we do indeed have multiple index entries - SSTableReader sstable = cfStore.getSSTables().iterator().next(); - RowIndexEntry indexEntry = sstable.getPosition(key, SSTableReader.Operator.EQ); + SSTableReader sstable = cfs.getSSTables().iterator().next(); + RowIndexEntry indexEntry = sstable.getPosition(Util.dk("0"), SSTableReader.Operator.EQ); assert indexEntry.columnsIndex().size() > 2; - validateSliceLarge(cfStore); + validateSliceLarge(cfs); } @Test - public void testLimitSSTables() throws CharacterCodingException + public void testLimitSSTables() throws Throwable { - Keyspace keyspace = Keyspace.open(KEYSPACE1); - ColumnFamilyStore cfStore = keyspace.getColumnFamilyStore("Standard1"); - cfStore.disableAutoCompaction(); - DecoratedKey key = Util.dk("row_maxmin"); - for (int j = 0; j < 10; j++) - { - ColumnFamily cf = ArrayBackedSortedColumns.factory.create(KEYSPACE1, "Standard1"); - for (int i = 1000 + (j*100); i < 1000 + ((j+1)*100); i++) - { - cf.addColumn(column("col" + i, ("v" + i), i)); - } - Mutation rm = new Mutation(KEYSPACE1, key.getKey(), cf); - rm.applyUnsafe(); - cfStore.forceBlockingFlush(); - } - ((ClearableHistogram)cfStore.metric.sstablesPerReadHistogram.cf).clear(); - ColumnFamily cf = cfStore.getColumnFamily(key, Composites.EMPTY, cellname("col1499"), false, 1000, System.currentTimeMillis()); - assertEquals(cfStore.metric.sstablesPerReadHistogram.cf.getSnapshot().getMax(), 5, 0.1); - int i = 0; - for (Cell c : cf.getSortedColumns()) - { - assertEquals(ByteBufferUtil.string(c.name().toByteBuffer()), "col" + (1000 + i++)); - } - assertEquals(i, 500); - ((ClearableHistogram)cfStore.metric.sstablesPerReadHistogram.cf).clear(); - cf = cfStore.getColumnFamily(key, cellname("col1500"), cellname("col2000"), false, 1000, System.currentTimeMillis()); - assertEquals(cfStore.metric.sstablesPerReadHistogram.cf.getSnapshot().getMax(), 5, 0.1); - - for (Cell c : cf.getSortedColumns()) - { - assertEquals(ByteBufferUtil.string(c.name().toByteBuffer()), "col"+(1000 + i++)); - } - assertEquals(i, 1000); - - // reverse - ((ClearableHistogram)cfStore.metric.sstablesPerReadHistogram.cf).clear(); - cf = cfStore.getColumnFamily(key, cellname("col2000"), cellname("col1500"), true, 1000, System.currentTimeMillis()); - assertEquals(cfStore.metric.sstablesPerReadHistogram.cf.getSnapshot().getMax(), 5, 0.1); - i = 500; - for (Cell c : cf.getSortedColumns()) - { - assertEquals(ByteBufferUtil.string(c.name().toByteBuffer()), "col"+(1000 + i++)); - } - assertEquals(i, 1000); - - } - - @Test - public void testLimitSSTablesComposites() - { - /* - creates 10 sstables, composite columns like this: - --------------------- - k |a0:0|a1:1|..|a9:9 - --------------------- - --------------------- - k |a0:10|a1:11|..|a9:19 - --------------------- - ... - --------------------- - k |a0:90|a1:91|..|a9:99 - --------------------- - then we slice out col1 = a5 and col2 > 85 -> which should let us just check 2 sstables and get 2 columns - */ - Keyspace keyspace = Keyspace.open(KEYSPACE1); - - ColumnFamilyStore cfs = keyspace.getColumnFamilyStore("StandardComposite2"); + String tableName = createTable("CREATE TABLE %s (a text, b int, c int, PRIMARY KEY (a, b))"); + final ColumnFamilyStore cfs = Keyspace.open(KEYSPACE).getColumnFamilyStore(tableName); + cfs.disableAutoCompaction(); + + for (int j = 0; j < 10; j++) + { + for (int i = 1000 + (j*100); i < 1000 + ((j+1)*100); i++) + execute("INSERT INTO %s (a, b, c) VALUES (?, ?, ?) USING TIMESTAMP ?", "0", i, i, (long)i); + + cfs.forceBlockingFlush(); + } + + ((ClearableHistogram)cfs.metric.sstablesPerReadHistogram.cf).clear(); + + SinglePartitionSliceCommand command = singlePartitionSlice(cfs, "0", slices(cfs, null, 1499, false), 1000); + int[] expectedValues = new int[500]; + for (int i = 0; i < 500; i++) + expectedValues[i] = i + 1000; + assertRowsInResult(cfs, command, expectedValues); + + assertEquals(5, cfs.metric.sstablesPerReadHistogram.cf.getSnapshot().getMax(), 0.1); + ((ClearableHistogram)cfs.metric.sstablesPerReadHistogram.cf).clear(); + + command = singlePartitionSlice(cfs, "0", slices(cfs, 1500, 2000, false), 1000); + for (int i = 0; i < 500; i++) + expectedValues[i] = i + 1500; + assertRowsInResult(cfs, command, expectedValues); + + assertEquals(5, cfs.metric.sstablesPerReadHistogram.cf.getSnapshot().getMax(), 0.1); + ((ClearableHistogram)cfs.metric.sstablesPerReadHistogram.cf).clear(); + + // reverse + command = singlePartitionSlice(cfs, "0", slices(cfs, 1500, 2000, true), 1000); + for (int i = 0; i < 500; i++) + expectedValues[i] = 1999 - i; + assertRowsInResult(cfs, command, expectedValues); + } + + @Test + public void testLimitSSTablesComposites() throws Throwable + { + // creates 10 sstables, composite columns like this: + // --------------------- + // k |a0:0|a1:1|..|a9:9 + // --------------------- + // --------------------- + // k |a0:10|a1:11|..|a9:19 + // --------------------- + // ... + // --------------------- + // k |a0:90|a1:91|..|a9:99 + // --------------------- + // then we slice out col1 = a5 and col2 > 85 -> which should let us just check 2 sstables and get 2 columns + String tableName = createTable("CREATE TABLE %s (a text, b text, c int, d int, PRIMARY KEY (a, b, c))"); + final ColumnFamilyStore cfs = Keyspace.open(KEYSPACE).getColumnFamilyStore(tableName); cfs.disableAutoCompaction(); - CellNameType type = cfs.getComparator(); - DecoratedKey key = Util.dk("k"); for (int j = 0; j < 10; j++) { for (int i = 0; i < 10; i++) - { - Mutation rm = new Mutation(KEYSPACE1, key.getKey()); - CellName colName = type.makeCellName(ByteBufferUtil.bytes("a" + i), ByteBufferUtil.bytes(j*10 + i)); - rm.add("StandardComposite2", colName, ByteBufferUtil.EMPTY_BYTE_BUFFER, 0); - rm.applyUnsafe(); - } + execute("INSERT INTO %s (a, b, c, d) VALUES (?, ?, ?, ?)", "0", "a" + i, j * 10 + i, 0); + cfs.forceBlockingFlush(); } - Composite start = type.builder().add(ByteBufferUtil.bytes("a5")).add(ByteBufferUtil.bytes(85)).build(); - Composite finish = type.builder().add(ByteBufferUtil.bytes("a5")).build().end(); + ((ClearableHistogram)cfs.metric.sstablesPerReadHistogram.cf).clear(); - ColumnFamily cf = cfs.getColumnFamily(key, start, finish, false, 1000, System.currentTimeMillis()); - int colCount = 0; - for (Cell c : cf) - colCount++; - assertEquals(2, colCount); + assertRows(execute("SELECT * FROM %s WHERE a = ? AND (b, c) >= (?, ?) AND (b) <= (?) LIMIT 1000", "0", "a5", 85, "a5"), + row("0", "a5", 85, 0), + row("0", "a5", 95, 0)); assertEquals(2, cfs.metric.sstablesPerReadHistogram.cf.getSnapshot().getMax(), 0.1); } - private void validateSliceLarge(ColumnFamilyStore cfStore) throws IOException + private void validateSliceLarge(ColumnFamilyStore cfs) { - DecoratedKey key = Util.dk("row3"); - ColumnFamily cf; - cf = cfStore.getColumnFamily(key, cellname("col1000"), Composites.EMPTY, false, 3, System.currentTimeMillis()); - assertColumns(cf, "col1000", "col1001", "col1002"); + ClusteringIndexSliceFilter filter = slices(cfs, 1000, null, false); + SinglePartitionSliceCommand command = new SinglePartitionSliceCommand( + cfs.metadata, FBUtilities.nowInSeconds(), ColumnFilter.all(cfs.metadata), RowFilter.NONE, DataLimits.cqlLimits(3), Util.dk("0"), filter); + assertRowsInResult(cfs, command, 1000, 1001, 1002); - ByteBuffer col; - col = cf.getColumn(cellname("col1000")).value(); - assertEquals(ByteBufferUtil.string(col), "v1000"); - col = cf.getColumn(cellname("col1001")).value(); - assertEquals(ByteBufferUtil.string(col), "v1001"); - col = cf.getColumn(cellname("col1002")).value(); - assertEquals(ByteBufferUtil.string(col), "v1002"); + filter = slices(cfs, 1195, null, false); + command = new SinglePartitionSliceCommand( + cfs.metadata, FBUtilities.nowInSeconds(), ColumnFilter.all(cfs.metadata), RowFilter.NONE, DataLimits.cqlLimits(3), Util.dk("0"), filter); + assertRowsInResult(cfs, command, 1195, 1196, 1197); - cf = cfStore.getColumnFamily(key, cellname("col1195"), Composites.EMPTY, false, 3, System.currentTimeMillis()); - assertColumns(cf, "col1195", "col1196", "col1197"); + filter = slices(cfs, null, 1996, true); + command = new SinglePartitionSliceCommand( + cfs.metadata, FBUtilities.nowInSeconds(), ColumnFilter.all(cfs.metadata), RowFilter.NONE, DataLimits.cqlLimits(1000), Util.dk("0"), filter); + int[] expectedValues = new int[997]; + for (int i = 0, v = 1996; v >= 1000; i++, v--) + expectedValues[i] = v; + assertRowsInResult(cfs, command, expectedValues); - col = cf.getColumn(cellname("col1195")).value(); - assertEquals(ByteBufferUtil.string(col), "v1195"); - col = cf.getColumn(cellname("col1196")).value(); - assertEquals(ByteBufferUtil.string(col), "v1196"); - col = cf.getColumn(cellname("col1197")).value(); - assertEquals(ByteBufferUtil.string(col), "v1197"); + filter = slices(cfs, 1990, null, false); + command = new SinglePartitionSliceCommand( + cfs.metadata, FBUtilities.nowInSeconds(), ColumnFilter.all(cfs.metadata), RowFilter.NONE, DataLimits.cqlLimits(3), Util.dk("0"), filter); + assertRowsInResult(cfs, command, 1990, 1991, 1992); + filter = slices(cfs, null, null, true); + command = new SinglePartitionSliceCommand( + cfs.metadata, FBUtilities.nowInSeconds(), ColumnFilter.all(cfs.metadata), RowFilter.NONE, DataLimits.cqlLimits(3), Util.dk("0"), filter); + assertRowsInResult(cfs, command, 1999, 1998, 1997); - cf = cfStore.getColumnFamily(key, cellname("col1996"), Composites.EMPTY, true, 1000, System.currentTimeMillis()); - Cell[] cells = cf.getSortedColumns().toArray(new Cell[0]); - for (int i = 1000; i < 1996; i++) - { - String expectedName = "col" + i; - Cell cell = cells[i - 1000]; - assertEquals(ByteBufferUtil.string(cell.name().toByteBuffer()), expectedName); - assertEquals(ByteBufferUtil.string(cell.value()), ("v" + i)); - } + filter = slices(cfs, null, 9000, true); + command = new SinglePartitionSliceCommand( + cfs.metadata, FBUtilities.nowInSeconds(), ColumnFilter.all(cfs.metadata), RowFilter.NONE, DataLimits.cqlLimits(3), Util.dk("0"), filter); + assertRowsInResult(cfs, command, 1999, 1998, 1997); - cf = cfStore.getColumnFamily(key, cellname("col1990"), Composites.EMPTY, false, 3, System.currentTimeMillis()); - assertColumns(cf, "col1990", "col1991", "col1992"); - col = cf.getColumn(cellname("col1990")).value(); - assertEquals(ByteBufferUtil.string(col), "v1990"); - col = cf.getColumn(cellname("col1991")).value(); - assertEquals(ByteBufferUtil.string(col), "v1991"); - col = cf.getColumn(cellname("col1992")).value(); - assertEquals(ByteBufferUtil.string(col), "v1992"); - - cf = cfStore.getColumnFamily(key, Composites.EMPTY, Composites.EMPTY, true, 3, System.currentTimeMillis()); - assertColumns(cf, "col1997", "col1998", "col1999"); - col = cf.getColumn(cellname("col1997")).value(); - assertEquals(ByteBufferUtil.string(col), "v1997"); - col = cf.getColumn(cellname("col1998")).value(); - assertEquals(ByteBufferUtil.string(col), "v1998"); - col = cf.getColumn(cellname("col1999")).value(); - assertEquals(ByteBufferUtil.string(col), "v1999"); - - cf = cfStore.getColumnFamily(key, cellname("col9000"), Composites.EMPTY, true, 3, System.currentTimeMillis()); - assertColumns(cf, "col1997", "col1998", "col1999"); - - cf = cfStore.getColumnFamily(key, cellname("col9000"), Composites.EMPTY, false, 3, System.currentTimeMillis()); - assertColumns(cf); - } - - public static void assertColumns(ColumnFamily container, String... columnNames) - { - Collection cells = container == null ? new TreeSet() : container.getSortedColumns(); - List L = new ArrayList(); - for (Cell cell : cells) - { - L.add(Util.string(cell.name().toByteBuffer())); - } - - List names = new ArrayList(columnNames.length); - - names.addAll(Arrays.asList(columnNames)); - - String[] columnNames1 = names.toArray(new String[0]); - String[] la = L.toArray(new String[cells.size()]); - - assert Arrays.equals(la, columnNames1) - : String.format("Columns [%s])] is not expected [%s]", - ((container == null) ? "" : CellNames.getColumnsString(container.getComparator(), cells)), - StringUtils.join(columnNames1, ",")); - } - - public static void assertColumn(ColumnFamily cf, String name, String value, long timestamp) - { - assertColumn(cf.getColumn(cellname(name)), value, timestamp); - } - - public static void assertColumn(Cell cell, String value, long timestamp) - { - assertNotNull(cell); - assertEquals(0, ByteBufferUtil.compareUnsigned(cell.value(), ByteBufferUtil.bytes(value))); - assertEquals(timestamp, cell.timestamp()); + filter = slices(cfs, 9000, null, false); + command = new SinglePartitionSliceCommand( + cfs.metadata, FBUtilities.nowInSeconds(), ColumnFilter.all(cfs.metadata), RowFilter.NONE, DataLimits.cqlLimits(3), Util.dk("0"), filter); + assertRowsInResult(cfs, command); } } diff --git a/test/unit/org/apache/cassandra/db/MultiKeyspaceTest.java b/test/unit/org/apache/cassandra/db/MultiKeyspaceTest.java new file mode 100644 index 0000000000..d69025372b --- /dev/null +++ b/test/unit/org/apache/cassandra/db/MultiKeyspaceTest.java @@ -0,0 +1,49 @@ +package org.apache.cassandra.db; +/* + * + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + * + */ + +import org.apache.cassandra.cql3.CQLTester; +import org.junit.Test; + + +public class MultiKeyspaceTest extends CQLTester +{ + @Test + public void testSameTableNames() throws Throwable + { + schemaChange("CREATE KEYSPACE multikstest1 WITH replication = {'class': 'SimpleStrategy', 'replication_factor': '1'}"); + schemaChange("CREATE TABLE multikstest1.standard1 (a int PRIMARY KEY, b int)"); + + schemaChange("CREATE KEYSPACE multikstest2 WITH replication = {'class': 'SimpleStrategy', 'replication_factor': '1'}"); + schemaChange("CREATE TABLE multikstest2.standard1 (a int PRIMARY KEY, b int)"); + + execute("INSERT INTO multikstest1.standard1 (a, b) VALUES (0, 0)"); + execute("INSERT INTO multikstest2.standard1 (a, b) VALUES (0, 0)"); + + Keyspace.open("multikstest1").flush(); + Keyspace.open("multikstest2").flush(); + + assertRows(execute("SELECT * FROM multikstest1.standard1"), + row(0, 0)); + assertRows(execute("SELECT * FROM multikstest2.standard1"), + row(0, 0)); + } +} diff --git a/test/unit/org/apache/cassandra/db/MultitableTest.java b/test/unit/org/apache/cassandra/db/MultitableTest.java deleted file mode 100644 index fd04b76ec4..0000000000 --- a/test/unit/org/apache/cassandra/db/MultitableTest.java +++ /dev/null @@ -1,80 +0,0 @@ -package org.apache.cassandra.db; -/* - * - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - * - */ - -import org.junit.BeforeClass; -import org.junit.Test; - -import static org.apache.cassandra.db.KeyspaceTest.assertColumns; -import org.apache.cassandra.SchemaLoader; -import org.apache.cassandra.Util; -import static org.apache.cassandra.Util.column; -import org.apache.cassandra.config.KSMetaData; -import org.apache.cassandra.exceptions.ConfigurationException; -import org.apache.cassandra.locator.SimpleStrategy; - -public class MultitableTest -{ - private static final String KEYSPACE1 = "MultitableTest1"; - private static final String KEYSPACE2 = "MultitableTest2"; - private static final String CF1 = "Standard1"; - - @BeforeClass - public static void defineSchema() throws ConfigurationException - { - SchemaLoader.prepareServer(); - SchemaLoader.createKeyspace(KEYSPACE1, - SimpleStrategy.class, - KSMetaData.optsWithRF(1), - SchemaLoader.standardCFMD(KEYSPACE1, CF1)); - SchemaLoader.createKeyspace(KEYSPACE2, - SimpleStrategy.class, - KSMetaData.optsWithRF(1), - SchemaLoader.standardCFMD(KEYSPACE2, CF1)); - } - - @Test - public void testSameCFs() - { - Keyspace keyspace1 = Keyspace.open(KEYSPACE1); - Keyspace keyspace2 = Keyspace.open(KEYSPACE2); - - Mutation rm; - DecoratedKey dk = Util.dk("keymulti"); - ColumnFamily cf; - - cf = ArrayBackedSortedColumns.factory.create(KEYSPACE1, "Standard1"); - cf.addColumn(column("col1", "val1", 1L)); - rm = new Mutation(KEYSPACE1, dk.getKey(), cf); - rm.applyUnsafe(); - - cf = ArrayBackedSortedColumns.factory.create(KEYSPACE2, "Standard1"); - cf.addColumn(column("col2", "val2", 1L)); - rm = new Mutation(KEYSPACE2, dk.getKey(), cf); - rm.applyUnsafe(); - - keyspace1.getColumnFamilyStore("Standard1").forceBlockingFlush(); - keyspace2.getColumnFamilyStore("Standard1").forceBlockingFlush(); - - assertColumns(Util.getColumnFamily(keyspace1, dk, "Standard1"), "col1"); - assertColumns(Util.getColumnFamily(keyspace2, dk, "Standard1"), "col2"); - } -} diff --git a/test/unit/org/apache/cassandra/db/NameSortTest.java b/test/unit/org/apache/cassandra/db/NameSortTest.java index c4361d863d..6391fcdfc6 100644 --- a/test/unit/org/apache/cassandra/db/NameSortTest.java +++ b/test/unit/org/apache/cassandra/db/NameSortTest.java @@ -18,20 +18,23 @@ */ package org.apache.cassandra.db; -import static org.junit.Assert.assertEquals; -import static org.apache.cassandra.Util.addMutation; import java.io.IOException; import java.nio.ByteBuffer; -import java.util.Collection; import org.apache.cassandra.SchemaLoader; import org.apache.cassandra.Util; +import org.apache.cassandra.config.ColumnDefinition; import org.apache.cassandra.config.KSMetaData; -import org.apache.cassandra.db.marshal.LongType; +import org.apache.cassandra.db.rows.Row; +import org.apache.cassandra.db.rows.RowIterator; +import org.apache.cassandra.db.partitions.*; +import org.apache.cassandra.db.marshal.AsciiType; +import org.apache.cassandra.db.partitions.PartitionIterator; import org.apache.cassandra.exceptions.ConfigurationException; import org.apache.cassandra.locator.SimpleStrategy; import org.apache.cassandra.utils.ByteBufferUtil; +import static org.junit.Assert.assertEquals; import org.junit.BeforeClass; import org.junit.Test; @@ -39,7 +42,6 @@ public class NameSortTest { private static final String KEYSPACE1 = "NameSortTest"; private static final String CF = "Standard1"; - private static final String CFSUPER = "Super1"; @BeforeClass public static void defineSchema() throws ConfigurationException @@ -48,8 +50,7 @@ public class NameSortTest SchemaLoader.createKeyspace(KEYSPACE1, SimpleStrategy.class, KSMetaData.optsWithRF(1), - SchemaLoader.standardCFMD(KEYSPACE1, CF), - SchemaLoader.superCFMD(KEYSPACE1, CFSUPER, LongType.instance)); + SchemaLoader.standardCFMD(KEYSPACE1, CF, 1000, AsciiType.instance)); } @Test @@ -76,56 +77,35 @@ public class NameSortTest private void testNameSort(int N) throws IOException { Keyspace keyspace = Keyspace.open(KEYSPACE1); - - for (int i = 0; i < N; ++i) + ColumnFamilyStore cfs = keyspace.getColumnFamilyStore(CF); + for (int i = 0; i < N; i++) { ByteBuffer key = ByteBufferUtil.bytes(Integer.toString(i)); - Mutation rm; - - // standard - for (int j = 0; j < 8; ++j) - { - ByteBuffer bytes = j % 2 == 0 ? ByteBufferUtil.bytes("a") : ByteBufferUtil.bytes("b"); - rm = new Mutation(KEYSPACE1, key); - rm.add("Standard1", Util.cellname("Cell-" + j), bytes, j); - rm.applyUnsafe(); - } - - // super - for (int j = 0; j < 8; ++j) - { - rm = new Mutation(KEYSPACE1, key); - for (int k = 0; k < 4; ++k) - { - String value = (j + k) % 2 == 0 ? "a" : "b"; - addMutation(rm, CFSUPER, "SuperColumn-" + j, k, value, k); - } - rm.applyUnsafe(); - } + RowUpdateBuilder rub = new RowUpdateBuilder(cfs.metadata, 0, key); + rub.clustering("cc"); + for (int j = 0; j < 8; j++) + rub.add("val" + j, j % 2 == 0 ? "a" : "b"); + rub.build().applyUnsafe(); } - - validateNameSort(keyspace, N); - + validateNameSort(cfs); keyspace.getColumnFamilyStore("Standard1").forceBlockingFlush(); - keyspace.getColumnFamilyStore(CFSUPER).forceBlockingFlush(); - validateNameSort(keyspace, N); + validateNameSort(cfs); } - private void validateNameSort(Keyspace keyspace, int N) throws IOException + private void validateNameSort(ColumnFamilyStore cfs) throws IOException { - for (int i = 0; i < N; ++i) + for (FilteredPartition partition : Util.getAll(Util.cmd(cfs).build())) { - DecoratedKey key = Util.dk(Integer.toString(i)); - ColumnFamily cf; - - cf = Util.getColumnFamily(keyspace, key, "Standard1"); - Collection cells = cf.getSortedColumns(); - for (Cell cell : cells) + for (Row r : partition) { - String name = ByteBufferUtil.string(cell.name().toByteBuffer()); - int j = Integer.valueOf(name.substring(name.length() - 1)); - byte[] bytes = j % 2 == 0 ? "a".getBytes() : "b".getBytes(); - assertEquals(new String(bytes), ByteBufferUtil.string(cell.value())); + for (ColumnDefinition cd : r.columns()) + { + if (r.getCell(cd) == null) + continue; + int cellVal = Integer.valueOf(cd.name.toString().substring(cd.name.toString().length() - 1)); + String expected = cellVal % 2 == 0 ? "a" : "b"; + assertEquals(expected, ByteBufferUtil.string(r.getCell(cd).value())); + } } } } diff --git a/test/unit/org/apache/cassandra/db/NativeCellTest.java b/test/unit/org/apache/cassandra/db/NativeCellTest.java deleted file mode 100644 index 6a2bf7358b..0000000000 --- a/test/unit/org/apache/cassandra/db/NativeCellTest.java +++ /dev/null @@ -1,251 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.apache.cassandra.db; - -import java.io.ByteArrayInputStream; -import java.io.DataInputStream; -import java.io.IOException; -import java.nio.ByteBuffer; -import java.security.MessageDigest; -import java.security.NoSuchAlgorithmException; -import java.util.Arrays; -import java.util.Random; -import java.util.concurrent.ThreadLocalRandom; - -import org.junit.Assert; -import org.junit.Test; - -import org.apache.cassandra.config.CFMetaData; -import org.apache.cassandra.config.ColumnDefinition; -import org.apache.cassandra.cql3.ColumnIdentifier; -import org.apache.cassandra.db.composites.CellName; -import org.apache.cassandra.db.composites.CellNameType; -import org.apache.cassandra.db.composites.CompoundDenseCellNameType; -import org.apache.cassandra.db.composites.CompoundSparseCellNameType; -import org.apache.cassandra.db.composites.SimpleDenseCellNameType; -import org.apache.cassandra.db.composites.SimpleSparseCellNameType; -import org.apache.cassandra.db.context.CounterContext; -import org.apache.cassandra.db.marshal.AbstractType; -import org.apache.cassandra.db.marshal.UTF8Type; -import org.apache.cassandra.exceptions.ConfigurationException; -import org.apache.cassandra.io.util.DataOutputBuffer; -import org.apache.cassandra.utils.concurrent.OpOrder; -import org.apache.cassandra.utils.memory.NativeAllocator; -import org.apache.cassandra.utils.memory.NativePool; - -import static org.apache.cassandra.db.composites.CellNames.compositeDense; -import static org.apache.cassandra.db.composites.CellNames.compositeSparse; -import static org.apache.cassandra.db.composites.CellNames.simpleDense; -import static org.apache.cassandra.db.composites.CellNames.simpleSparse; -import static org.apache.cassandra.utils.ByteBufferUtil.bytes; - -public class NativeCellTest -{ - - private static final NativeAllocator nativeAllocator = new NativePool(Integer.MAX_VALUE, Integer.MAX_VALUE, 1f, null).newAllocator(); - private static final OpOrder.Group group = new OpOrder().start(); - - static class Name - { - final CellName name; - final CellNameType type; - Name(CellName name, CellNameType type) - { - this.name = name; - this.type = type; - } - } - - static ByteBuffer[] bytess(String ... strings) - { - ByteBuffer[] r = new ByteBuffer[strings.length]; - for (int i = 0 ; i < r.length ; i++) - r[i] = bytes(strings[i]); - return r; - } - - final static Name[] TESTS = new Name[] - { - new Name(simpleDense(bytes("a")), new SimpleDenseCellNameType(UTF8Type.instance)), - new Name(simpleSparse(new ColumnIdentifier("a", true)), new SimpleSparseCellNameType(UTF8Type.instance)), - new Name(compositeDense(bytes("a"), bytes("b")), new CompoundDenseCellNameType(Arrays.>asList(UTF8Type.instance, UTF8Type.instance))), - new Name(compositeSparse(bytess("b", "c"), new ColumnIdentifier("a", true), false), new CompoundSparseCellNameType(Arrays.>asList(UTF8Type.instance, UTF8Type.instance))), - new Name(compositeSparse(bytess("b", "c"), new ColumnIdentifier("a", true), true), new CompoundSparseCellNameType(Arrays.>asList(UTF8Type.instance, UTF8Type.instance))) - }; - - private static final CFMetaData metadata = new CFMetaData("", "", ColumnFamilyType.Standard, null); - static - { - try - { - metadata.addColumnDefinition(new ColumnDefinition(null, null, new ColumnIdentifier("a", true), UTF8Type.instance, null, null, null, null, null)); - } - catch (ConfigurationException e) - { - throw new AssertionError(); - } - } - - @Test - public void testCells() throws IOException - { - Random rand = ThreadLocalRandom.current(); - for (Name test : TESTS) - { - byte[] bytes = new byte[16]; - rand.nextBytes(bytes); - - // test regular Cell - Cell buf, nat; - buf = new BufferCell(test.name, ByteBuffer.wrap(bytes), rand.nextLong()); - nat = buf.localCopy(metadata, nativeAllocator, group); - test(test, buf, nat); - - // test DeletedCell - buf = new BufferDeletedCell(test.name, rand.nextInt(100000), rand.nextLong()); - nat = buf.localCopy(metadata, nativeAllocator, group); - test(test, buf, nat); - - // test ExpiringCell - buf = new BufferExpiringCell(test.name, ByteBuffer.wrap(bytes), rand.nextLong(), rand.nextInt(100000)); - nat = buf.localCopy(metadata, nativeAllocator, group); - test(test, buf, nat); - - // test CounterCell - buf = new BufferCounterCell(test.name, CounterContext.instance().createLocal(rand.nextLong()), rand.nextLong(), rand.nextInt(100000)); - nat = buf.localCopy(metadata, nativeAllocator, group); - test(test, buf, nat); - } - } - - - @Test - public void testComparator() - { - - Random rand = ThreadLocalRandom.current(); - for (Name test : TESTS) - { - byte[] bytes = new byte[7]; - byte[] bytes2 = new byte[7]; - rand.nextBytes(bytes); - rand.nextBytes(bytes2); - - // test regular Cell - Cell buf, nat, buf2, nat2; - buf = new BufferCell(test.name, ByteBuffer.wrap(bytes), rand.nextLong()); - nat = buf.localCopy(metadata, nativeAllocator, group); - - buf2 = new BufferCell(test.name, ByteBuffer.wrap(bytes2), rand.nextLong()); - nat2 = buf2.localCopy(metadata, nativeAllocator, group); - - assert test.type.compare(buf.name(), nat.name()) == 0; - assert test.type.compare(buf2.name(), nat2.name()) == 0; - - int val = test.type.compare(buf.name(), buf2.name()); - assert test.type.compare(nat.name(), nat2.name()) == val; - assert test.type.compare(nat.name(), buf2.name()) == val; - assert test.type.compare(buf.name(), nat2.name()) == val; - - - // test DeletedCell - buf = new BufferDeletedCell(test.name, rand.nextInt(100000), rand.nextLong()); - nat = buf.localCopy(metadata, nativeAllocator, group); - buf2 = new BufferDeletedCell(test.name, rand.nextInt(100000), rand.nextLong()); - nat2 = buf2.localCopy(metadata, nativeAllocator, group); - - assert test.type.compare(buf.name(), nat.name()) == 0; - assert test.type.compare(buf2.name(), nat2.name()) == 0; - - val = test.type.compare(buf.name(), buf2.name()); - assert test.type.compare(nat.name(), nat2.name()) == val; - assert test.type.compare(nat.name(), buf2.name()) == val; - assert test.type.compare(buf.name(), nat2.name()) == val; - - - - // test ExpiringCell - buf = new BufferExpiringCell(test.name, ByteBuffer.wrap(bytes), rand.nextLong(), rand.nextInt(100000)); - nat = buf.localCopy(metadata, nativeAllocator, group); - - buf2 = new BufferExpiringCell(test.name, ByteBuffer.wrap(bytes2), rand.nextLong(), rand.nextInt(100000)); - nat2 = buf2.localCopy(metadata, nativeAllocator, group); - - assert test.type.compare(buf.name(), nat.name()) == 0; - assert test.type.compare(buf2.name(), nat2.name()) == 0; - - val = test.type.compare(buf.name(), buf2.name()); - assert test.type.compare(nat.name(), nat2.name()) == val; - assert test.type.compare(nat.name(), buf2.name()) == val; - assert test.type.compare(buf.name(), nat2.name()) == val; - - - // test CounterCell - buf = new BufferCounterCell(test.name, CounterContext.instance().createLocal(rand.nextLong()), rand.nextLong(), rand.nextInt(100000)); - nat = buf.localCopy(metadata, nativeAllocator, group); - - buf2 = new BufferCounterCell(test.name, CounterContext.instance().createLocal(rand.nextLong()), rand.nextLong(), rand.nextInt(100000)); - nat2 = buf2.localCopy(metadata, nativeAllocator, group); - - assert test.type.compare(buf.name(), nat.name()) == 0; - assert test.type.compare(buf2.name(), nat2.name()) == 0; - - val = test.type.compare(buf.name(), buf2.name()); - assert test.type.compare(nat.name(), nat2.name()) == val; - assert test.type.compare(nat.name(), buf2.name()) == val; - assert test.type.compare(buf.name(), nat2.name()) == val; - - } - } - - static void test(Name test, Cell buf, Cell nat) throws IOException - { - Assert.assertTrue(buf.equals(nat)); - Assert.assertTrue(nat.equals(buf)); - Assert.assertTrue(buf.equals(buf)); - Assert.assertTrue(nat.equals(nat)); - - try - { - MessageDigest d1 = MessageDigest.getInstance("MD5"); - MessageDigest d2 = MessageDigest.getInstance("MD5"); - buf.updateDigest(d1); - nat.updateDigest(d2); - Assert.assertArrayEquals(d1.digest(), d2.digest()); - } - catch (NoSuchAlgorithmException e) - { - throw new IllegalStateException(e); - } - - byte[] serialized; - try (DataOutputBuffer bufOut = new DataOutputBuffer()) - { - test.type.columnSerializer().serialize(nat, bufOut); - serialized = bufOut.getData(); - } - - ByteArrayInputStream bufIn = new ByteArrayInputStream(serialized, 0, serialized.length); - Cell deserialized = test.type.columnSerializer().deserialize(new DataInputStream(bufIn)); - Assert.assertTrue(buf.equals(deserialized)); - - } - - - -} diff --git a/test/unit/org/apache/cassandra/db/PartitionRangeReadTest.java b/test/unit/org/apache/cassandra/db/PartitionRangeReadTest.java new file mode 100644 index 0000000000..d293745955 --- /dev/null +++ b/test/unit/org/apache/cassandra/db/PartitionRangeReadTest.java @@ -0,0 +1,375 @@ +/* +* Licensed to the Apache Software Foundation (ASF) under one +* or more contributor license agreements. See the NOTICE file +* distributed with this work for additional information +* regarding copyright ownership. The ASF licenses this file +* to you under the Apache License, Version 2.0 (the +* "License"); you may not use this file except in compliance +* with the License. You may obtain a copy of the License at +* +* http://www.apache.org/licenses/LICENSE-2.0 +* +* Unless required by applicable law or agreed to in writing, +* software distributed under the License is distributed on an +* "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +* KIND, either express or implied. See the License for the +* specific language governing permissions and limitations +* under the License. +*/ +package org.apache.cassandra.db; + +import java.io.File; +import java.math.BigInteger; +import java.nio.ByteBuffer; +import java.nio.charset.CharacterCodingException; +import java.util.Collection; +import java.util.List; + +import org.junit.BeforeClass; +import org.junit.Test; +import com.google.common.collect.Iterators; + +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertTrue; + +import org.apache.cassandra.*; +import org.apache.cassandra.config.ColumnDefinition; +import org.apache.cassandra.config.KSMetaData; +import org.apache.cassandra.db.rows.Row; +import org.apache.cassandra.db.rows.RowIterator; +import org.apache.cassandra.db.marshal.IntegerType; +import org.apache.cassandra.db.partitions.*; +import org.apache.cassandra.exceptions.ConfigurationException; +import org.apache.cassandra.io.sstable.Component; +import org.apache.cassandra.io.sstable.format.SSTableReader; +import org.apache.cassandra.locator.SimpleStrategy; +import org.apache.cassandra.utils.ByteBufferUtil; +import org.apache.cassandra.utils.FBUtilities; + +public class PartitionRangeReadTest +{ + public static final String KEYSPACE1 = "PartitionRangeReadTest1"; + public static final String KEYSPACE2 = "PartitionRangeReadTest2"; + public static final String CF_STANDARD1 = "Standard1"; + public static final String CF_STANDARDINT = "StandardInteger1"; + + @BeforeClass + public static void defineSchema() throws ConfigurationException + { + SchemaLoader.prepareServer(); + SchemaLoader.createKeyspace(KEYSPACE1, + SimpleStrategy.class, + KSMetaData.optsWithRF(1), + SchemaLoader.standardCFMD(KEYSPACE1, CF_STANDARD1), + SchemaLoader.denseCFMD(KEYSPACE1, CF_STANDARDINT, IntegerType.instance)); + SchemaLoader.createKeyspace(KEYSPACE2, + SimpleStrategy.class, + KSMetaData.optsWithRF(1), + SchemaLoader.standardCFMD(KEYSPACE2, CF_STANDARD1)); + } + + @Test + public void testInclusiveBounds() + { + ColumnFamilyStore cfs = Keyspace.open(KEYSPACE2).getColumnFamilyStore(CF_STANDARD1); + new RowUpdateBuilder(cfs.metadata, 0, ByteBufferUtil.bytes("key1")) + .clustering("cc1") + .add("val", "asdf").build().applyUnsafe(); + new RowUpdateBuilder(cfs.metadata, 0, ByteBufferUtil.bytes("key2")) + .clustering("cc2") + .add("val", "asdf").build().applyUnsafe(); + + assertEquals(2, Util.getAll(Util.cmd(cfs).fromIncl("cc1").toIncl("cc2").build()).size()); + } + + @Test + public void testCassandra6778() throws CharacterCodingException + { + String cfname = CF_STANDARDINT; + Keyspace keyspace = Keyspace.open(KEYSPACE1); + ColumnFamilyStore cfs = keyspace.getColumnFamilyStore(cfname); + cfs.truncateBlocking(); + + ByteBuffer col = ByteBufferUtil.bytes("val"); + ColumnDefinition cDef = cfs.metadata.getColumnDefinition(col); + + // insert two columns that represent the same integer but have different binary forms (the + // second one is padded with extra zeros) + new RowUpdateBuilder(cfs.metadata, 0, "k1") + .clustering(new BigInteger(new byte[]{1})) + .add("val", "val1") + .build() + .applyUnsafe(); + cfs.forceBlockingFlush(); + + new RowUpdateBuilder(cfs.metadata, 1, "k1") + .clustering(new BigInteger(new byte[]{0, 0, 1})) + .add("val", "val2") + .build() + .applyUnsafe(); + cfs.forceBlockingFlush(); + + // fetch by the first column name; we should get the second version of the column value + Row row = Util.getOnlyRow(Util.cmd(cfs, "k1").includeRow(new BigInteger(new byte[]{1})).build()); + assertTrue(row.getCell(cDef).value().equals(ByteBufferUtil.bytes("val2"))); + + // fetch by the second column name; we should get the second version of the column value + row = Util.getOnlyRow(Util.cmd(cfs, "k1").includeRow(new BigInteger(new byte[]{0, 0, 1})).build()); + assertTrue(row.getCell(cDef).value().equals(ByteBufferUtil.bytes("val2"))); + } + + @Test + public void testRangeSliceInclusionExclusion() throws Throwable + { + String keyspaceName = KEYSPACE1; + String cfName = CF_STANDARD1; + Keyspace keyspace = Keyspace.open(keyspaceName); + ColumnFamilyStore cfs = keyspace.getColumnFamilyStore(cfName); + cfs.clearUnsafe(); + + for (int i = 0; i < 10; ++i) + { + RowUpdateBuilder builder = new RowUpdateBuilder(cfs.metadata, 10, String.valueOf(i)); + builder.clustering("c"); + builder.add("val", String.valueOf(i)); + builder.build().applyUnsafe(); + } + + cfs.forceBlockingFlush(); + + ColumnDefinition cDef = cfs.metadata.getColumnDefinition(ByteBufferUtil.bytes("val")); + + List partitions; + + // Start and end inclusive + partitions = Util.getAll(Util.cmd(cfs).fromKeyIncl("2").toKeyIncl("7").build()); + assertEquals(6, partitions.size()); + assertTrue(partitions.get(0).iterator().next().getCell(cDef).value().equals(ByteBufferUtil.bytes("2"))); + assertTrue(partitions.get(partitions.size() - 1).iterator().next().getCell(cDef).value().equals(ByteBufferUtil.bytes("7"))); + + // Start and end excluded + partitions = Util.getAll(Util.cmd(cfs).fromKeyExcl("2").toKeyExcl("7").build()); + assertEquals(4, partitions.size()); + assertTrue(partitions.get(0).iterator().next().getCell(cDef).value().equals(ByteBufferUtil.bytes("3"))); + assertTrue(partitions.get(partitions.size() - 1).iterator().next().getCell(cDef).value().equals(ByteBufferUtil.bytes("6"))); + + // Start excluded, end included + partitions = Util.getAll(Util.cmd(cfs).fromKeyExcl("2").toKeyIncl("7").build()); + assertEquals(5, partitions.size()); + assertTrue(partitions.get(0).iterator().next().getCell(cDef).value().equals(ByteBufferUtil.bytes("3"))); + assertTrue(partitions.get(partitions.size() - 1).iterator().next().getCell(cDef).value().equals(ByteBufferUtil.bytes("7"))); + + // Start included, end excluded + partitions = Util.getAll(Util.cmd(cfs).fromKeyIncl("2").toKeyExcl("7").build()); + assertEquals(5, partitions.size()); + assertTrue(partitions.get(0).iterator().next().getCell(cDef).value().equals(ByteBufferUtil.bytes("2"))); + assertTrue(partitions.get(partitions.size() - 1).iterator().next().getCell(cDef).value().equals(ByteBufferUtil.bytes("6"))); + } + + // TODO: Port or remove, depending on what DataLimits.thriftLimits (per cell) looks like +// @Test +// public void testRangeSliceColumnsLimit() throws Throwable +// { +// String keyspaceName = KEYSPACE1; +// String cfName = CF_STANDARD1; +// Keyspace keyspace = Keyspace.open(keyspaceName); +// ColumnFamilyStore cfs = keyspace.getColumnFamilyStore(cfName); +// cfs.clearUnsafe(); +// +// Cell[] cols = new Cell[5]; +// for (int i = 0; i < 5; i++) +// cols[i] = column("c" + i, "value", 1); +// +// putColsStandard(cfs, Util.dk("a"), cols[0], cols[1], cols[2], cols[3], cols[4]); +// putColsStandard(cfs, Util.dk("b"), cols[0], cols[1]); +// putColsStandard(cfs, Util.dk("c"), cols[0], cols[1], cols[2], cols[3]); +// cfs.forceBlockingFlush(); +// +// SlicePredicate sp = new SlicePredicate(); +// sp.setSlice_range(new SliceRange()); +// sp.getSlice_range().setCount(1); +// sp.getSlice_range().setStart(ArrayUtils.EMPTY_BYTE_ARRAY); +// sp.getSlice_range().setFinish(ArrayUtils.EMPTY_BYTE_ARRAY); +// +// assertTotalColCount(cfs.getRangeSlice(Util.range("", ""), +// null, +// ThriftValidation.asIFilter(sp, cfs.metadata, null), +// 3, +// System.currentTimeMillis(), +// true, +// false), +// 3); +// assertTotalColCount(cfs.getRangeSlice(Util.range("", ""), +// null, +// ThriftValidation.asIFilter(sp, cfs.metadata, null), +// 5, +// System.currentTimeMillis(), +// true, +// false), +// 5); +// assertTotalColCount(cfs.getRangeSlice(Util.range("", ""), +// null, +// ThriftValidation.asIFilter(sp, cfs.metadata, null), +// 8, +// System.currentTimeMillis(), +// true, +// false), +// 8); +// assertTotalColCount(cfs.getRangeSlice(Util.range("", ""), +// null, +// ThriftValidation.asIFilter(sp, cfs.metadata, null), +// 10, +// System.currentTimeMillis(), +// true, +// false), +// 10); +// assertTotalColCount(cfs.getRangeSlice(Util.range("", ""), +// null, +// ThriftValidation.asIFilter(sp, cfs.metadata, null), +// 100, +// System.currentTimeMillis(), +// true, +// false), +// 11); +// +// // Check that when querying by name, we always include all names for a +// // gien row even if it means returning more columns than requested (this is necesseray for CQL) +// sp = new SlicePredicate(); +// sp.setColumn_names(Arrays.asList( +// ByteBufferUtil.bytes("c0"), +// ByteBufferUtil.bytes("c1"), +// ByteBufferUtil.bytes("c2") +// )); +// +// assertTotalColCount(cfs.getRangeSlice(Util.range("", ""), +// null, +// ThriftValidation.asIFilter(sp, cfs.metadata, null), +// 1, +// System.currentTimeMillis(), +// true, +// false), +// 3); +// assertTotalColCount(cfs.getRangeSlice(Util.range("", ""), +// null, +// ThriftValidation.asIFilter(sp, cfs.metadata, null), +// 4, +// System.currentTimeMillis(), +// true, +// false), +// 5); +// assertTotalColCount(cfs.getRangeSlice(Util.range("", ""), +// null, +// ThriftValidation.asIFilter(sp, cfs.metadata, null), +// 5, +// System.currentTimeMillis(), +// true, +// false), +// 5); +// assertTotalColCount(cfs.getRangeSlice(Util.range("", ""), +// null, +// ThriftValidation.asIFilter(sp, cfs.metadata, null), +// 6, +// System.currentTimeMillis(), +// true, +// false), +// 8); +// assertTotalColCount(cfs.getRangeSlice(Util.range("", ""), +// null, +// ThriftValidation.asIFilter(sp, cfs.metadata, null), +// 100, +// System.currentTimeMillis(), +// true, +// false), +// 8); +// } + + // TODO: Port or remove, depending on what DataLimits.thriftLimits (per cell) looks like +// @Test +// public void testRangeSlicePaging() throws Throwable +// { +// String keyspaceName = KEYSPACE1; +// String cfName = CF_STANDARD1; +// Keyspace keyspace = Keyspace.open(keyspaceName); +// ColumnFamilyStore cfs = keyspace.getColumnFamilyStore(cfName); +// cfs.clearUnsafe(); +// +// Cell[] cols = new Cell[4]; +// for (int i = 0; i < 4; i++) +// cols[i] = column("c" + i, "value", 1); +// +// DecoratedKey ka = Util.dk("a"); +// DecoratedKey kb = Util.dk("b"); +// DecoratedKey kc = Util.dk("c"); +// +// PartitionPosition min = Util.rp(""); +// +// putColsStandard(cfs, ka, cols[0], cols[1], cols[2], cols[3]); +// putColsStandard(cfs, kb, cols[0], cols[1], cols[2]); +// putColsStandard(cfs, kc, cols[0], cols[1], cols[2], cols[3]); +// cfs.forceBlockingFlush(); +// +// SlicePredicate sp = new SlicePredicate(); +// sp.setSlice_range(new SliceRange()); +// sp.getSlice_range().setCount(1); +// sp.getSlice_range().setStart(ArrayUtils.EMPTY_BYTE_ARRAY); +// sp.getSlice_range().setFinish(ArrayUtils.EMPTY_BYTE_ARRAY); +// +// Collection rows; +// Row row, row1, row2; +// IDiskAtomFilter filter = ThriftValidation.asIFilter(sp, cfs.metadata, null); +// +// rows = cfs.getRangeSlice(cfs.makeExtendedFilter(Util.range("", ""), filter, null, 3, true, true, System.currentTimeMillis())); +// assert rows.size() == 1 : "Expected 1 row, got " + toString(rows); +// row = rows.iterator().next(); +// assertColumnNames(row, "c0", "c1", "c2"); +// +// sp.getSlice_range().setStart(ByteBufferUtil.getArray(ByteBufferUtil.bytes("c2"))); +// filter = ThriftValidation.asIFilter(sp, cfs.metadata, null); +// rows = cfs.getRangeSlice(cfs.makeExtendedFilter(new Bounds(ka, min), filter, null, 3, true, true, System.currentTimeMillis())); +// assert rows.size() == 2 : "Expected 2 rows, got " + toString(rows); +// Iterator iter = rows.iterator(); +// row1 = iter.next(); +// row2 = iter.next(); +// assertColumnNames(row1, "c2", "c3"); +// assertColumnNames(row2, "c0"); +// +// sp.getSlice_range().setStart(ByteBufferUtil.getArray(ByteBufferUtil.bytes("c0"))); +// filter = ThriftValidation.asIFilter(sp, cfs.metadata, null); +// rows = cfs.getRangeSlice(cfs.makeExtendedFilter(new Bounds(row2.key, min), filter, null, 3, true, true, System.currentTimeMillis())); +// assert rows.size() == 1 : "Expected 1 row, got " + toString(rows); +// row = rows.iterator().next(); +// assertColumnNames(row, "c0", "c1", "c2"); +// +// sp.getSlice_range().setStart(ByteBufferUtil.getArray(ByteBufferUtil.bytes("c2"))); +// filter = ThriftValidation.asIFilter(sp, cfs.metadata, null); +// rows = cfs.getRangeSlice(cfs.makeExtendedFilter(new Bounds(row.key, min), filter, null, 3, true, true, System.currentTimeMillis())); +// assert rows.size() == 2 : "Expected 2 rows, got " + toString(rows); +// iter = rows.iterator(); +// row1 = iter.next(); +// row2 = iter.next(); +// assertColumnNames(row1, "c2"); +// assertColumnNames(row2, "c0", "c1"); +// +// // Paging within bounds +// SliceQueryFilter sf = new SliceQueryFilter(cellname("c1"), +// cellname("c2"), +// false, +// 0); +// rows = cfs.getRangeSlice(cfs.makeExtendedFilter(new Bounds(ka, kc), sf, cellname("c2"), cellname("c1"), null, 2, true, System.currentTimeMillis())); +// assert rows.size() == 2 : "Expected 2 rows, got " + toString(rows); +// iter = rows.iterator(); +// row1 = iter.next(); +// row2 = iter.next(); +// assertColumnNames(row1, "c2"); +// assertColumnNames(row2, "c1"); +// +// rows = cfs.getRangeSlice(cfs.makeExtendedFilter(new Bounds(kb, kc), sf, cellname("c1"), cellname("c1"), null, 10, true, System.currentTimeMillis())); +// assert rows.size() == 2 : "Expected 2 rows, got " + toString(rows); +// iter = rows.iterator(); +// row1 = iter.next(); +// row2 = iter.next(); +// assertColumnNames(row1, "c1", "c2"); +// assertColumnNames(row2, "c1"); +// } +} + diff --git a/test/unit/org/apache/cassandra/db/PartitionTest.java b/test/unit/org/apache/cassandra/db/PartitionTest.java new file mode 100644 index 0000000000..24dfc6d10d --- /dev/null +++ b/test/unit/org/apache/cassandra/db/PartitionTest.java @@ -0,0 +1,180 @@ +/* +* Licensed to the Apache Software Foundation (ASF) under one +* or more contributor license agreements. See the NOTICE file +* distributed with this work for additional information +* regarding copyright ownership. The ASF licenses this file +* to you under the Apache License, Version 2.0 (the +* "License"); you may not use this file except in compliance +* with the License. You may obtain a copy of the License at +* +* http://www.apache.org/licenses/LICENSE-2.0 +* +* Unless required by applicable law or agreed to in writing, +* software distributed under the License is distributed on an +* "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +* KIND, either express or implied. See the License for the +* specific language governing permissions and limitations +* under the License. +*/ +package org.apache.cassandra.db; + +import java.io.IOException; +import java.security.MessageDigest; +import java.security.NoSuchAlgorithmException; +import java.util.Arrays; + +import org.junit.BeforeClass; +import org.junit.Test; + +import org.apache.cassandra.config.ColumnDefinition; +import org.apache.cassandra.config.KSMetaData; +import org.apache.cassandra.db.rows.UnfilteredRowIterator; +import org.apache.cassandra.db.rows.UnfilteredRowIterators; +import org.apache.cassandra.db.rows.RowStats; +import org.apache.cassandra.db.marshal.AsciiType; +import org.apache.cassandra.db.marshal.BytesType; +import org.apache.cassandra.db.partitions.*; +import org.apache.cassandra.io.util.DataOutputBuffer; +import org.apache.cassandra.locator.SimpleStrategy; +import org.apache.cassandra.net.MessagingService; +import org.apache.cassandra.exceptions.ConfigurationException; +import org.apache.cassandra.SchemaLoader; +import org.apache.cassandra.Util; +import org.apache.cassandra.utils.ByteBufferUtil; +import org.apache.cassandra.utils.FBUtilities; +import org.apache.hadoop.io.DataInputBuffer; + +import static junit.framework.Assert.assertTrue; +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertFalse; + +public class PartitionTest +{ + static int version = MessagingService.current_version; + private static final String KEYSPACE1 = "Keyspace1"; + private static final String CF_STANDARD1 = "Standard1"; + private static final String CF_TENCOL = "TenColumns"; + private static final String CF_COUNTER1 = "Counter1"; + + @BeforeClass + public static void defineSchema() throws ConfigurationException + { + SchemaLoader.prepareServer(); + SchemaLoader.createKeyspace(KEYSPACE1, + SimpleStrategy.class, + KSMetaData.optsWithRF(1), + SchemaLoader.standardCFMD(KEYSPACE1, CF_STANDARD1), + SchemaLoader.standardCFMD(KEYSPACE1, CF_TENCOL, 10, AsciiType.instance), + SchemaLoader.denseCFMD(KEYSPACE1, CF_COUNTER1, BytesType.instance)); + } + + @Test + public void testSingleColumn() throws IOException + { + ColumnFamilyStore cfs = Keyspace.open(KEYSPACE1).getColumnFamilyStore(CF_STANDARD1); + PartitionUpdate update = new RowUpdateBuilder(cfs.metadata, 5, "key1") + .clustering("c") + .add("val", "val1") + .buildUpdate(); + + ArrayBackedCachedPartition partition = ArrayBackedCachedPartition.create(update.unfilteredIterator(), FBUtilities.nowInSeconds()); + + DataOutputBuffer bufOut = new DataOutputBuffer(); + CachedPartition.cacheSerializer.serialize(partition, bufOut); + + DataInputBuffer bufIn = new DataInputBuffer(); + bufIn.reset(bufOut.getData(), 0, bufOut.getLength()); + CachedPartition deserialized = CachedPartition.cacheSerializer.deserialize(bufIn); + + assert deserialized != null; + assert deserialized.metadata().cfName.equals(CF_STANDARD1); + assert deserialized.partitionKey().equals(partition.partitionKey()); + } + + @Test + public void testManyColumns() throws IOException + { + ColumnFamilyStore cfs = Keyspace.open(KEYSPACE1).getColumnFamilyStore(CF_TENCOL); + RowUpdateBuilder builder = new RowUpdateBuilder(cfs.metadata, 5, "key1") + .clustering("c") + .add("val", "val1"); + + for (int i = 0; i < 10; i++) + builder.add("val" + i, "val" + i); + + PartitionUpdate update = builder.buildUpdate(); + + ArrayBackedCachedPartition partition = ArrayBackedCachedPartition.create(update.unfilteredIterator(), FBUtilities.nowInSeconds()); + + DataOutputBuffer bufOut = new DataOutputBuffer(); + CachedPartition.cacheSerializer.serialize(partition, bufOut); + + DataInputBuffer bufIn = new DataInputBuffer(); + bufIn.reset(bufOut.getData(), 0, bufOut.getLength()); + CachedPartition deserialized = CachedPartition.cacheSerializer.deserialize(bufIn); + + assertEquals(partition.columns().regulars.columnCount(), deserialized.columns().regulars.columnCount()); + assertTrue(deserialized.columns().regulars.getSimple(1).equals(partition.columns().regulars.getSimple(1))); + assertTrue(deserialized.columns().regulars.getSimple(5).equals(partition.columns().regulars.getSimple(5))); + + ColumnDefinition cDef = cfs.metadata.getColumnDefinition(ByteBufferUtil.bytes("val8")); + assertTrue(partition.lastRow().getCell(cDef).value().equals(deserialized.lastRow().getCell(cDef).value())); + assert deserialized.partitionKey().equals(partition.partitionKey()); + } + + @Test + public void testDigest() throws NoSuchAlgorithmException + { + ColumnFamilyStore cfs = Keyspace.open(KEYSPACE1).getColumnFamilyStore(CF_TENCOL); + RowUpdateBuilder builder = new RowUpdateBuilder(cfs.metadata, 5, "key1").clustering("c").add("val", "val1"); + for (int i = 0; i < 10; i++) + builder.add("val" + i, "val" + i); + builder.build().applyUnsafe(); + + new RowUpdateBuilder(cfs.metadata, 5, "key2").clustering("c").add("val", "val2").build().applyUnsafe(); + + ArrayBackedPartition p1 = Util.getOnlyPartitionUnfiltered(Util.cmd(cfs, "key1").build()); + ArrayBackedPartition p2 = Util.getOnlyPartitionUnfiltered(Util.cmd(cfs, "key2").build()); + + MessageDigest digest1 = MessageDigest.getInstance("MD5"); + MessageDigest digest2 = MessageDigest.getInstance("MD5"); + UnfilteredRowIterators.digest(p1.unfilteredIterator(), digest1); + UnfilteredRowIterators.digest(p2.unfilteredIterator(), digest2); + assertFalse(Arrays.equals(digest1.digest(), digest2.digest())); + + p1 = Util.getOnlyPartitionUnfiltered(Util.cmd(cfs, "key2").build()); + p2 = Util.getOnlyPartitionUnfiltered(Util.cmd(cfs, "key2").build()); + digest1 = MessageDigest.getInstance("MD5"); + digest2 = MessageDigest.getInstance("MD5"); + UnfilteredRowIterators.digest(p1.unfilteredIterator(), digest1); + UnfilteredRowIterators.digest(p2.unfilteredIterator(), digest2); + assertTrue(Arrays.equals(digest1.digest(), digest2.digest())); + + p1 = Util.getOnlyPartitionUnfiltered(Util.cmd(cfs, "key2").build()); + RowUpdateBuilder.deleteRow(cfs.metadata, 6, "key2", "c").applyUnsafe(); + p2 = Util.getOnlyPartitionUnfiltered(Util.cmd(cfs, "key2").build()); + digest1 = MessageDigest.getInstance("MD5"); + digest2 = MessageDigest.getInstance("MD5"); + UnfilteredRowIterators.digest(p1.unfilteredIterator(), digest1); + UnfilteredRowIterators.digest(p2.unfilteredIterator(), digest2); + assertFalse(Arrays.equals(digest1.digest(), digest2.digest())); + } + + @Test + public void testColumnStatsRecordsRowDeletesCorrectly() + { + long timestamp = System.currentTimeMillis(); + int localDeletionTime = (int) (timestamp / 1000); + + ColumnFamilyStore cfs = Keyspace.open(KEYSPACE1).getColumnFamilyStore(CF_TENCOL); + RowUpdateBuilder builder = new RowUpdateBuilder(cfs.metadata, 5, "key1").clustering("c").add("val", "val1"); + for (int i = 0; i < 10; i++) + builder.add("val" + i, "val" + i); + builder.build().applyUnsafe(); + + RowUpdateBuilder.deleteRow(cfs.metadata, 10, "key1", "c").applyUnsafe(); + ArrayBackedPartition partition = Util.getOnlyPartitionUnfiltered(Util.cmd(cfs, "key1").build()); + RowStats stats = partition.stats(); + assertEquals(localDeletionTime, stats.minLocalDeletionTime); + } +} diff --git a/test/unit/org/apache/cassandra/db/RangeTombstoneListTest.java b/test/unit/org/apache/cassandra/db/RangeTombstoneListTest.java index 7dc7300760..73b91ea380 100644 --- a/test/unit/org/apache/cassandra/db/RangeTombstoneListTest.java +++ b/test/unit/org/apache/cassandra/db/RangeTombstoneListTest.java @@ -18,190 +18,20 @@ */ package org.apache.cassandra.db; +import java.nio.ByteBuffer; import java.util.*; import org.junit.Test; import static org.junit.Assert.*; import org.apache.cassandra.Util; -import org.apache.cassandra.db.composites.*; +import org.apache.cassandra.db.*; import org.apache.cassandra.db.marshal.IntegerType; import org.apache.cassandra.utils.ByteBufferUtil; public class RangeTombstoneListTest { - private static final Comparator cmp = new SimpleDenseCellNameType(IntegerType.instance); - - @Test - public void testDiff() - { - RangeTombstoneList superset; - RangeTombstoneList subset; - RangeTombstoneList diff; - Iterator iter; - - // no difference - superset = new RangeTombstoneList(cmp, 10); - subset = new RangeTombstoneList(cmp, 10); - superset.add(rt(1, 10, 10)); - superset.add(rt(20, 30, 10)); - superset.add(rt(40, 50, 10)); - subset.add(rt(1, 10, 10)); - subset.add(rt(20, 30, 10)); - subset.add(rt(40, 50, 10)); - assertNull( subset.diff(superset)); - - // all items in subset are contained by the first range in the superset - superset = new RangeTombstoneList(cmp, 10); - subset = new RangeTombstoneList(cmp, 10); - subset.add(rt(1, 2, 3)); - subset.add(rt(3, 4, 4)); - subset.add(rt(5, 6, 5)); - superset.add(rt(1, 10, 10)); - superset.add(rt(20, 30, 10)); - superset.add(rt(40, 50, 10)); - diff = subset.diff(superset); - iter = diff.iterator(); - assertRT(rt(1, 10, 10), iter.next()); - assertRT(rt(20, 30, 10), iter.next()); - assertRT(rt(40, 50, 10), iter.next()); - assertFalse(iter.hasNext()); - - // multiple subset RTs are contained by superset RTs - superset = new RangeTombstoneList(cmp, 10); - subset = new RangeTombstoneList(cmp, 10); - subset.add(rt(1, 2, 1)); - subset.add(rt(3, 4, 2)); - subset.add(rt(5, 6, 3)); - superset.add(rt(1, 5, 2)); - superset.add(rt(5, 6, 3)); - superset.add(rt(6, 10, 2)); - diff = subset.diff(superset); - iter = diff.iterator(); - assertRT(rt(1, 5, 2), iter.next()); - assertRT(rt(6, 10, 2), iter.next()); - assertFalse(iter.hasNext()); - - // the superset has one RT that covers the entire subset - superset = new RangeTombstoneList(cmp, 10); - subset = new RangeTombstoneList(cmp, 10); - superset.add(rt(1, 50, 10)); - subset.add(rt(1, 10, 10)); - subset.add(rt(20, 30, 10)); - subset.add(rt(40, 50, 10)); - diff = subset.diff(superset); - iter = diff.iterator(); - assertRT(rt(1, 50, 10), iter.next()); - assertFalse(iter.hasNext()); - - // the superset has one RT that covers the remainder of the subset - superset = new RangeTombstoneList(cmp, 10); - subset = new RangeTombstoneList(cmp, 10); - superset.add(rt(1, 10, 10)); - superset.add(rt(20, 50, 10)); - subset.add(rt(1, 10, 10)); - subset.add(rt(20, 30, 10)); - subset.add(rt(40, 50, 10)); - diff = subset.diff(superset); - iter = diff.iterator(); - assertRT(rt(20, 50, 10), iter.next()); - assertFalse(iter.hasNext()); - - // only the timestamp differs on one RT - superset = new RangeTombstoneList(cmp, 10); - subset = new RangeTombstoneList(cmp, 10); - superset.add(rt(1, 10, 10)); - superset.add(rt(20, 30, 20)); - superset.add(rt(40, 50, 10)); - subset.add(rt(1, 10, 10)); - subset.add(rt(20, 30, 10)); - subset.add(rt(40, 50, 10)); - diff = subset.diff(superset); - iter = diff.iterator(); - assertRT(rt(20, 30, 20), iter.next()); - assertFalse(iter.hasNext()); - - // superset has a large range on an RT at the start - superset = new RangeTombstoneList(cmp, 10); - subset = new RangeTombstoneList(cmp, 10); - superset.add(rt(1, 10, 10)); - superset.add(rt(20, 30, 10)); - superset.add(rt(40, 50, 10)); - subset.add(rt(1, 2, 3)); - subset.add(rt(20, 30, 10)); - subset.add(rt(40, 50, 10)); - diff = subset.diff(superset); - iter = diff.iterator(); - assertRT(rt(1, 10, 10), iter.next()); - assertFalse(iter.hasNext()); - - // superset has a larger range on an RT in the middle - superset = new RangeTombstoneList(cmp, 10); - subset = new RangeTombstoneList(cmp, 10); - superset.add(rt(1, 10, 10)); - superset.add(rt(20, 30, 10)); - superset.add(rt(40, 50, 10)); - subset.add(rt(1, 10, 10)); - subset.add(rt(20, 25, 10)); - subset.add(rt(40, 50, 10)); - diff = subset.diff(superset); - iter = diff.iterator(); - assertRT(rt(20, 30, 10), iter.next()); - assertFalse(iter.hasNext()); - - // superset has a larger range on an RT at the end - superset = new RangeTombstoneList(cmp, 10); - subset = new RangeTombstoneList(cmp, 10); - superset.add(rt(1, 10, 10)); - superset.add(rt(20, 30, 10)); - superset.add(rt(40, 55, 10)); - subset.add(rt(1, 10, 10)); - subset.add(rt(20, 30, 10)); - subset.add(rt(40, 50, 10)); - diff = subset.diff(superset); - iter = diff.iterator(); - assertRT(rt(40, 55, 10), iter.next()); - assertFalse(iter.hasNext()); - - // superset has one additional RT in the middle - superset = new RangeTombstoneList(cmp, 10); - subset = new RangeTombstoneList(cmp, 10); - superset.add(rt(1, 10, 10)); - superset.add(rt(20, 30, 10)); - superset.add(rt(40, 50, 10)); - subset.add(rt(1, 10, 10)); - subset.add(rt(40, 50, 10)); - diff = subset.diff(superset); - iter = diff.iterator(); - assertRT(rt(20, 30, 10), iter.next()); - assertFalse(iter.hasNext()); - - // superset has one additional RT at the start - superset = new RangeTombstoneList(cmp, 10); - subset = new RangeTombstoneList(cmp, 10); - superset.add(rt(1, 10, 10)); - superset.add(rt(20, 30, 10)); - superset.add(rt(40, 50, 10)); - subset.add(rt(20, 30, 10)); - subset.add(rt(40, 50, 10)); - diff = subset.diff(superset); - iter = diff.iterator(); - assertRT(rt(1, 10, 10), iter.next()); - assertFalse(iter.hasNext()); - - // superset has one additional RT at the end - superset = new RangeTombstoneList(cmp, 10); - subset = new RangeTombstoneList(cmp, 10); - superset.add(rt(1, 10, 10)); - superset.add(rt(20, 30, 10)); - superset.add(rt(40, 50, 10)); - subset.add(rt(1, 10, 10)); - subset.add(rt(20, 30, 10)); - diff = subset.diff(superset); - iter = diff.iterator(); - assertRT(rt(40, 50, 10), iter.next()); - assertFalse(iter.hasNext()); - } + private static final ClusteringComparator cmp = new ClusteringComparator(IntegerType.instance); @Test public void sortedAdditionTest() @@ -224,7 +54,7 @@ public class RangeTombstoneListTest Iterator iter = l.iterator(); assertRT(rt1, iter.next()); assertRT(rt2, iter.next()); - assertRT(rt3, iter.next()); + assertRT(rtei(10, 13, 1), iter.next()); assert !iter.hasNext(); } @@ -250,7 +80,7 @@ public class RangeTombstoneListTest Iterator iter = l.iterator(); assertRT(rt1, iter.next()); assertRT(rt2, iter.next()); - assertRT(rt3, iter.next()); + assertRT(rtei(10, 13, 1), iter.next()); assert !iter.hasNext(); } @@ -272,18 +102,18 @@ public class RangeTombstoneListTest l.add(rt(0, 15, 1)); Iterator iter = l.iterator(); - assertRT(rt(0, 1, 1), iter.next()); - assertRT(rt(1, 4, 2), iter.next()); - assertRT(rt(4, 8, 3), iter.next()); + assertRT(rtie(0, 1, 1), iter.next()); + assertRT(rtie(1, 4, 2), iter.next()); + assertRT(rtie(4, 8, 3), iter.next()); assertRT(rt(8, 13, 4), iter.next()); - assertRT(rt(13, 15, 1), iter.next()); + assertRT(rtei(13, 15, 1), iter.next()); assert !iter.hasNext(); RangeTombstoneList l2 = new RangeTombstoneList(cmp, initialCapacity); l2.add(rt(4, 10, 12L)); l2.add(rt(0, 8, 25L)); - assertEquals(25L, l2.searchDeletionTime(b(8)).markedForDeleteAt); + assertEquals(25L, l2.searchDeletionTime(clustering(8)).markedForDeleteAt()); } @Test @@ -307,9 +137,9 @@ public class RangeTombstoneListTest l1.add(rt(3, 7, 5)); Iterator iter1 = l1.iterator(); - assertRT(rt(0, 3, 3), iter1.next()); + assertRT(rtie(0, 3, 3), iter1.next()); assertRT(rt(3, 7, 5), iter1.next()); - assertRT(rt(7, 10, 3), iter1.next()); + assertRT(rtei(7, 10, 3), iter1.next()); assert !iter1.hasNext(); RangeTombstoneList l2 = new RangeTombstoneList(cmp, 0); @@ -330,9 +160,9 @@ public class RangeTombstoneListTest l.add(rt(1, 4, 2)); l.add(rt(4, 10, 5)); - assertEquals(2, l.searchDeletionTime(b(3)).markedForDeleteAt); - assertEquals(5, l.searchDeletionTime(b(4)).markedForDeleteAt); - assertEquals(5, l.searchDeletionTime(b(8)).markedForDeleteAt); + assertEquals(2, l.searchDeletionTime(clustering(3)).markedForDeleteAt()); + assertEquals(5, l.searchDeletionTime(clustering(4)).markedForDeleteAt()); + assertEquals(5, l.searchDeletionTime(clustering(8)).markedForDeleteAt()); assertEquals(3, l.size()); } @@ -346,20 +176,20 @@ public class RangeTombstoneListTest l.add(rt(14, 15, 3)); l.add(rt(15, 17, 6)); - assertEquals(null, l.searchDeletionTime(b(-1))); + assertEquals(null, l.searchDeletionTime(clustering(-1))); - assertEquals(5, l.searchDeletionTime(b(0)).markedForDeleteAt); - assertEquals(5, l.searchDeletionTime(b(3)).markedForDeleteAt); - assertEquals(5, l.searchDeletionTime(b(4)).markedForDeleteAt); + assertEquals(5, l.searchDeletionTime(clustering(0)).markedForDeleteAt()); + assertEquals(5, l.searchDeletionTime(clustering(3)).markedForDeleteAt()); + assertEquals(5, l.searchDeletionTime(clustering(4)).markedForDeleteAt()); - assertEquals(2, l.searchDeletionTime(b(5)).markedForDeleteAt); + assertEquals(2, l.searchDeletionTime(clustering(5)).markedForDeleteAt()); - assertEquals(null, l.searchDeletionTime(b(7))); + assertEquals(null, l.searchDeletionTime(clustering(7))); - assertEquals(3, l.searchDeletionTime(b(14)).markedForDeleteAt); + assertEquals(3, l.searchDeletionTime(clustering(14)).markedForDeleteAt()); - assertEquals(6, l.searchDeletionTime(b(15)).markedForDeleteAt); - assertEquals(null, l.searchDeletionTime(b(18))); + assertEquals(6, l.searchDeletionTime(clustering(15)).markedForDeleteAt()); + assertEquals(null, l.searchDeletionTime(clustering(18))); } @Test @@ -379,13 +209,12 @@ public class RangeTombstoneListTest l1.addAll(l2); Iterator iter = l1.iterator(); - assertRT(rt(0, 3, 5), iter.next()); - assertRT(rt(3, 4, 7), iter.next()); - assertRT(rt(4, 5, 7), iter.next()); - assertRT(rt(6, 7, 2), iter.next()); + assertRT(rtie(0, 3, 5), iter.next()); + assertRT(rt(3, 5, 7), iter.next()); + assertRT(rtie(6, 7, 2), iter.next()); assertRT(rt(7, 8, 3), iter.next()); - assertRT(rt(8, 10, 2), iter.next()); - assertRT(rt(10, 12, 1), iter.next()); + assertRT(rtei(8, 10, 2), iter.next()); + assertRT(rtei(10, 12, 1), iter.next()); assertRT(rt(14, 17, 4), iter.next()); assert !iter.hasNext(); @@ -403,7 +232,7 @@ public class RangeTombstoneListTest l1.addAll(l2); Iterator iter = l1.iterator(); - assertRT(rt(3, 5, 2), iter.next()); + assertRT(rtie(3, 5, 2), iter.next()); assertRT(rt(5, 7, 7), iter.next()); assert !iter.hasNext(); @@ -428,55 +257,6 @@ public class RangeTombstoneListTest assert !iter.hasNext(); } - @Test - public void purgetTest() - { - RangeTombstoneList l = new RangeTombstoneList(cmp, 0); - l.add(rt(0, 4, 5, 110)); - l.add(rt(4, 6, 2, 98)); - l.add(rt(9, 12, 1, 200)); - l.add(rt(14, 15, 3, 3)); - l.add(rt(15, 17, 6, 45)); - - l.purge(100); - - Iterator iter = l.iterator(); - assertRT(rt(0, 4, 5, 110), iter.next()); - assertRT(rt(9, 12, 1, 200), iter.next()); - - assert !iter.hasNext(); - } - - @Test - public void minMaxTest() - { - RangeTombstoneList l = new RangeTombstoneList(cmp, 0); - l.add(rt(0, 4, 5, 110)); - l.add(rt(4, 6, 2, 98)); - l.add(rt(9, 12, 1, 200)); - l.add(rt(14, 15, 3, 3)); - l.add(rt(15, 17, 6, 45)); - - assertEquals(1, l.minMarkedAt()); - assertEquals(6, l.maxMarkedAt()); - } - - @Test - public void insertSameTest() - { - // Simple test that adding the same element multiple time ends up - // with that element only a single time (CASSANDRA-9485) - - RangeTombstoneList l = new RangeTombstoneList(cmp, 0); - l.add(rt(4, 4, 5, 100)); - l.add(rt(4, 4, 6, 110)); - l.add(rt(4, 4, 4, 90)); - - Iterator iter = l.iterator(); - assertRT(rt(4, 4, 6, 110), iter.next()); - assert !iter.hasNext(); - } - private RangeTombstoneList makeRandom(Random rand, int size, int maxItSize, int maxItDistance, int maxMarkedAt) { RangeTombstoneList l = new RangeTombstoneList(cmp, size); @@ -539,34 +319,33 @@ public class RangeTombstoneListTest private static void assertRT(RangeTombstone expected, RangeTombstone actual) { - assertEquals(String.format("Expected %s but got %s", toString(expected), toString(actual)), expected, actual); + assertTrue(String.format("%s != %s", toString(expected), toString(actual)), cmp.compare(expected.deletedSlice().start(), actual.deletedSlice().start()) == 0); + assertTrue(String.format("%s != %s", toString(expected), toString(actual)), cmp.compare(expected.deletedSlice().end(), actual.deletedSlice().end()) == 0); + assertEquals(String.format("%s != %s", toString(expected), toString(actual)), expected.deletionTime(), actual.deletionTime()); } private static void assertValid(RangeTombstoneList l) { - // We check that ranges are in the right order and that we never have something - // like ...[x, x][x, x] ... - int prevStart = -2; - int prevEnd = -1; - for (RangeTombstone rt : l) + if (l.isEmpty()) + return; + + // We check that ranges are in the right order and non overlapping + Iterator iter = l.iterator(); + Slice prev = iter.next().deletedSlice(); + assertFalse("Invalid empty slice " + prev.toString(cmp), prev.isEmpty(cmp)); + + while (iter.hasNext()) { - int curStart = i(rt.min); - int curEnd = i(rt.max); + Slice curr = iter.next().deletedSlice(); - assertTrue("Invalid " + toString(l), prevEnd <= curStart); - assertTrue("Invalid " + toString(l), curStart <= curEnd); - - if (curStart == curEnd && prevEnd == curStart) - assertTrue("Invalid " + toString(l), prevStart != prevEnd); - - prevStart = curStart; - prevEnd = curEnd; + assertFalse("Invalid empty slice " + curr.toString(cmp), curr.isEmpty(cmp)); + assertTrue("Slice not in order or overlapping : " + prev.toString(cmp) + curr.toString(cmp), cmp.compare(prev.end(), curr.start()) < 0); } } private static String toString(RangeTombstone rt) { - return String.format("[%d, %d]@%d", i(rt.min), i(rt.max), rt.data.markedForDeleteAt); + return String.format("%s@%d", rt.deletedSlice().toString(cmp), rt.deletionTime().markedForDeleteAt()); } private static String toString(RangeTombstoneList l) @@ -578,14 +357,19 @@ public class RangeTombstoneListTest return sb.append(" }").toString(); } - private static Composite b(int i) + private static Clustering clustering(int i) { - return Util.cellname(i); + return new SimpleClustering(bb(i)); } - private static int i(Composite c) + private static ByteBuffer bb(int i) { - return ByteBufferUtil.toInt(c.toByteBuffer()); + return ByteBufferUtil.bytes(i); + } + + private static int i(Slice.Bound bound) + { + return ByteBufferUtil.toInt(bound.get(0)); } private static RangeTombstone rt(int start, int end, long tstamp) @@ -595,6 +379,26 @@ public class RangeTombstoneListTest private static RangeTombstone rt(int start, int end, long tstamp, int delTime) { - return new RangeTombstone(b(start), b(end), tstamp, delTime); + return new RangeTombstone(Slice.make(Slice.Bound.inclusiveStartOf(bb(start)), Slice.Bound.inclusiveEndOf(bb(end))), new SimpleDeletionTime(tstamp, delTime)); + } + + private static RangeTombstone rtei(int start, int end, long tstamp) + { + return rtei(start, end, tstamp, 0); + } + + private static RangeTombstone rtei(int start, int end, long tstamp, int delTime) + { + return new RangeTombstone(Slice.make(Slice.Bound.exclusiveStartOf(bb(start)), Slice.Bound.inclusiveEndOf(bb(end))), new SimpleDeletionTime(tstamp, delTime)); + } + + private static RangeTombstone rtie(int start, int end, long tstamp) + { + return rtie(start, end, tstamp, 0); + } + + private static RangeTombstone rtie(int start, int end, long tstamp, int delTime) + { + return new RangeTombstone(Slice.make(Slice.Bound.inclusiveStartOf(bb(start)), Slice.Bound.exclusiveEndOf(bb(end))), new SimpleDeletionTime(tstamp, delTime)); } } diff --git a/test/unit/org/apache/cassandra/db/RangeTombstoneTest.java b/test/unit/org/apache/cassandra/db/RangeTombstoneTest.java index 9ce12362db..c5060e543f 100644 --- a/test/unit/org/apache/cassandra/db/RangeTombstoneTest.java +++ b/test/unit/org/apache/cassandra/db/RangeTombstoneTest.java @@ -23,41 +23,36 @@ import java.util.ArrayList; import java.util.Collection; import java.util.List; import java.util.Set; -import java.util.SortedSet; -import java.util.TreeSet; import java.util.concurrent.ExecutionException; import java.util.concurrent.Future; import com.google.common.collect.ImmutableMap; import com.google.common.collect.Iterators; -import org.apache.cassandra.io.sstable.ISSTableScanner; import org.apache.cassandra.io.sstable.format.SSTableReader; import org.junit.BeforeClass; import org.junit.Test; +import org.apache.cassandra.AbstractReadCommandBuilder; import org.apache.cassandra.SchemaLoader; import org.apache.cassandra.config.*; import org.apache.cassandra.Util; -import org.apache.cassandra.db.columniterator.OnDiskAtomIterator; +import org.apache.cassandra.UpdateBuilder; import org.apache.cassandra.db.compaction.CompactionManager; import org.apache.cassandra.db.compaction.SizeTieredCompactionStrategy; -import org.apache.cassandra.db.composites.CellName; -import org.apache.cassandra.db.composites.CellNames; -import org.apache.cassandra.db.composites.Composites; -import org.apache.cassandra.db.filter.ColumnSlice; -import org.apache.cassandra.db.filter.IDiskAtomFilter; -import org.apache.cassandra.db.filter.QueryFilter; -import org.apache.cassandra.db.filter.SliceQueryFilter; +import org.apache.cassandra.db.filter.*; +import org.apache.cassandra.db.rows.*; +import org.apache.cassandra.db.partitions.*; import org.apache.cassandra.db.index.PerColumnSecondaryIndex; import org.apache.cassandra.db.index.SecondaryIndex; import org.apache.cassandra.db.index.SecondaryIndexSearcher; +import org.apache.cassandra.db.marshal.UTF8Type; import org.apache.cassandra.db.marshal.Int32Type; -import org.apache.cassandra.db.marshal.IntegerType; import org.apache.cassandra.exceptions.ConfigurationException; import org.apache.cassandra.locator.SimpleStrategy; import org.apache.cassandra.io.sstable.metadata.StatsMetadata; import org.apache.cassandra.utils.ByteBufferUtil; +import org.apache.cassandra.utils.FBUtilities; import org.apache.cassandra.utils.concurrent.OpOrder; import static org.apache.cassandra.Util.dk; @@ -77,7 +72,7 @@ public class RangeTombstoneTest SchemaLoader.createKeyspace(KSNAME, SimpleStrategy.class, KSMetaData.optsWithRF(1), - SchemaLoader.standardCFMD(KSNAME, CFNAME, IntegerType.instance)); + SchemaLoader.standardCFMD(KSNAME, CFNAME, 0, UTF8Type.instance, Int32Type.instance, Int32Type.instance)); } @Test @@ -88,55 +83,49 @@ public class RangeTombstoneTest // Inserting data String key = "k1"; - Mutation rm; - ColumnFamily cf; - rm = new Mutation(KSNAME, ByteBufferUtil.bytes(key)); + UpdateBuilder builder; + + builder = UpdateBuilder.create(cfs.metadata, key).withTimestamp(0); for (int i = 0; i < 40; i += 2) - add(rm, i, 0); - rm.applyUnsafe(); + builder.newRow(i).add("val", i); + builder.applyUnsafe(); cfs.forceBlockingFlush(); - rm = new Mutation(KSNAME, ByteBufferUtil.bytes(key)); - cf = rm.addOrGet(CFNAME); - delete(cf, 10, 22, 1); - rm.applyUnsafe(); - cfs.forceBlockingFlush(); + new RowUpdateBuilder(cfs.metadata, 1, key).addRangeTombstone(10, 22).build().applyUnsafe(); - rm = new Mutation(KSNAME, ByteBufferUtil.bytes(key)); + builder = UpdateBuilder.create(cfs.metadata, key).withTimestamp(2); for (int i = 1; i < 40; i += 2) - add(rm, i, 2); - rm.applyUnsafe(); - cfs.forceBlockingFlush(); + builder.newRow(i).add("val", i); + builder.applyUnsafe(); - rm = new Mutation(KSNAME, ByteBufferUtil.bytes(key)); - cf = rm.addOrGet(CFNAME); - delete(cf, 19, 27, 3); - rm.applyUnsafe(); + new RowUpdateBuilder(cfs.metadata, 3, key).addRangeTombstone(19, 27).build().applyUnsafe(); // We don't flush to test with both a range tomsbtone in memtable and in sstable // Queries by name int[] live = new int[]{ 4, 9, 11, 17, 28 }; int[] dead = new int[]{ 12, 19, 21, 24, 27 }; - SortedSet columns = new TreeSet(cfs.getComparator()); + + AbstractReadCommandBuilder.SinglePartitionBuilder cmdBuilder = Util.cmd(cfs, key); for (int i : live) - columns.add(b(i)); + cmdBuilder.includeRow(i); for (int i : dead) - columns.add(b(i)); - cf = cfs.getColumnFamily(QueryFilter.getNamesFilter(dk(key), CFNAME, columns, System.currentTimeMillis())); + cmdBuilder.includeRow(i); + + Partition partition = Util.getOnlyPartitionUnfiltered(cmdBuilder.build()); for (int i : live) - assertTrue("Cell " + i + " should be live", isLive(cf, cf.getColumn(b(i)))); + assertTrue("Row " + i + " should be live", partition.getRow(new SimpleClustering(bb(i))).hasLiveData(FBUtilities.nowInSeconds())); for (int i : dead) - assertTrue("Cell " + i + " shouldn't be live", !isLive(cf, cf.getColumn(b(i)))); + assertTrue("Row " + i + " shouldn't be live", !partition.getRow(new SimpleClustering(bb(i))).hasLiveData(FBUtilities.nowInSeconds())); // Queries by slices - cf = cfs.getColumnFamily(QueryFilter.getSliceFilter(dk(key), CFNAME, b(7), b(30), false, Integer.MAX_VALUE, System.currentTimeMillis())); + partition = Util.getOnlyPartitionUnfiltered(Util.cmd(cfs, key).fromIncl(7).toIncl(30).build()); for (int i : new int[]{ 7, 8, 9, 11, 13, 15, 17, 28, 29, 30 }) - assertTrue("Cell " + i + " should be live", isLive(cf, cf.getColumn(b(i)))); + assertTrue("Row " + i + " should be live", partition.getRow(new SimpleClustering(bb(i))).hasLiveData(FBUtilities.nowInSeconds())); for (int i : new int[]{ 10, 12, 14, 16, 18, 19, 20, 21, 22, 23, 24, 25, 26, 27 }) - assertTrue("Cell " + i + " shouldn't be live", !isLive(cf, cf.getColumn(b(i)))); + assertTrue("Row " + i + " shouldn't be live", partition.getRow(new SimpleClustering(bb(i))) == null); } @Test @@ -148,141 +137,135 @@ public class RangeTombstoneTest // Inserting data String key = "k111"; - Mutation rm; - ColumnFamily cf; - rm = new Mutation(KSNAME, ByteBufferUtil.bytes(key)); + UpdateBuilder builder = UpdateBuilder.create(cfs.metadata, key).withTimestamp(0); for (int i = 0; i < 40; i += 2) - add(rm, i, 0); - rm.applyUnsafe(); + builder.newRow(i).add("val", i); + builder.applyUnsafe(); - rm = new Mutation(KSNAME, ByteBufferUtil.bytes(key)); - cf = rm.addOrGet(CFNAME); - delete(cf, 5, 10, 1); - rm.applyUnsafe(); + new RowUpdateBuilder(cfs.metadata, 1, key).addRangeTombstone(5, 10).build().applyUnsafe(); - rm = new Mutation(KSNAME, ByteBufferUtil.bytes(key)); - cf = rm.addOrGet(CFNAME); - delete(cf, 15, 20, 2); - rm.applyUnsafe(); + new RowUpdateBuilder(cfs.metadata, 2, key).addRangeTombstone(15, 20).build().applyUnsafe(); - cf = cfs.getColumnFamily(QueryFilter.getSliceFilter(dk(key), CFNAME, b(11), b(14), false, Integer.MAX_VALUE, System.currentTimeMillis())); - Collection rt = rangeTombstones(cf); + ArrayBackedPartition partition; + + partition = Util.getOnlyPartitionUnfiltered(Util.cmd(cfs, key).fromIncl(11).toIncl(14).build()); + Collection rt = rangeTombstones(partition); assertEquals(0, rt.size()); - cf = cfs.getColumnFamily(QueryFilter.getSliceFilter(dk(key), CFNAME, b(11), b(15), false, Integer.MAX_VALUE, System.currentTimeMillis())); - rt = rangeTombstones(cf); + partition = Util.getOnlyPartitionUnfiltered(Util.cmd(cfs, key).fromIncl(11).toIncl(15).build()); + rt = rangeTombstones(partition); assertEquals(1, rt.size()); - cf = cfs.getColumnFamily(QueryFilter.getSliceFilter(dk(key), CFNAME, b(20), b(25), false, Integer.MAX_VALUE, System.currentTimeMillis())); - rt = rangeTombstones(cf); + partition = Util.getOnlyPartitionUnfiltered(Util.cmd(cfs, key).fromIncl(20).toIncl(25).build()); + rt = rangeTombstones(partition); assertEquals(1, rt.size()); - cf = cfs.getColumnFamily(QueryFilter.getSliceFilter(dk(key), CFNAME, b(12), b(25), false, Integer.MAX_VALUE, System.currentTimeMillis())); - rt = rangeTombstones(cf); + partition = Util.getOnlyPartitionUnfiltered(Util.cmd(cfs, key).fromIncl(12).toIncl(25).build()); + rt = rangeTombstones(partition); assertEquals(1, rt.size()); - cf = cfs.getColumnFamily(QueryFilter.getSliceFilter(dk(key), CFNAME, b(25), b(35), false, Integer.MAX_VALUE, System.currentTimeMillis())); - rt = rangeTombstones(cf); + partition = Util.getOnlyPartitionUnfiltered(Util.cmd(cfs, key).fromIncl(25).toIncl(35).build()); + rt = rangeTombstones(partition); assertEquals(0, rt.size()); - cf = cfs.getColumnFamily(QueryFilter.getSliceFilter(dk(key), CFNAME, b(1), b(40), false, Integer.MAX_VALUE, System.currentTimeMillis())); - rt = rangeTombstones(cf); + partition = Util.getOnlyPartitionUnfiltered(Util.cmd(cfs, key).fromIncl(1).toIncl(40).build()); + rt = rangeTombstones(partition); assertEquals(2, rt.size()); - cf = cfs.getColumnFamily(QueryFilter.getSliceFilter(dk(key), CFNAME, b(7), b(17), false, Integer.MAX_VALUE, System.currentTimeMillis())); - rt = rangeTombstones(cf); + partition = Util.getOnlyPartitionUnfiltered(Util.cmd(cfs, key).fromIncl(7).toIncl(17).build()); + rt = rangeTombstones(partition); assertEquals(2, rt.size()); - cf = cfs.getColumnFamily(QueryFilter.getSliceFilter(dk(key), CFNAME, b(5), b(20), false, Integer.MAX_VALUE, System.currentTimeMillis())); - rt = rangeTombstones(cf); + partition = Util.getOnlyPartitionUnfiltered(Util.cmd(cfs, key).fromIncl(5).toIncl(20).build()); + rt = rangeTombstones(partition); assertEquals(2, rt.size()); - cf = cfs.getColumnFamily(QueryFilter.getSliceFilter(dk(key), CFNAME, b(5), b(15), false, Integer.MAX_VALUE, System.currentTimeMillis())); - rt = rangeTombstones(cf); + partition = Util.getOnlyPartitionUnfiltered(Util.cmd(cfs, key).fromIncl(5).toIncl(20).build()); + rt = rangeTombstones(partition); assertEquals(2, rt.size()); - cf = cfs.getColumnFamily(QueryFilter.getSliceFilter(dk(key), CFNAME, b(1), b(2), false, Integer.MAX_VALUE, System.currentTimeMillis())); - rt = rangeTombstones(cf); + partition = Util.getOnlyPartitionUnfiltered(Util.cmd(cfs, key).fromIncl(1).toIncl(2).build()); + rt = rangeTombstones(partition); assertEquals(0, rt.size()); - cf = cfs.getColumnFamily(QueryFilter.getSliceFilter(dk(key), CFNAME, b(1), b(5), false, Integer.MAX_VALUE, System.currentTimeMillis())); - rt = rangeTombstones(cf); + partition = Util.getOnlyPartitionUnfiltered(Util.cmd(cfs, key).fromIncl(1).toIncl(5).build()); + rt = rangeTombstones(partition); assertEquals(1, rt.size()); - cf = cfs.getColumnFamily(QueryFilter.getSliceFilter(dk(key), CFNAME, b(1), b(10), false, Integer.MAX_VALUE, System.currentTimeMillis())); - rt = rangeTombstones(cf); + partition = Util.getOnlyPartitionUnfiltered(Util.cmd(cfs, key).fromIncl(1).toIncl(10).build()); + rt = rangeTombstones(partition); assertEquals(1, rt.size()); - cf = cfs.getColumnFamily(QueryFilter.getSliceFilter(dk(key), CFNAME, b(5), b(6), false, Integer.MAX_VALUE, System.currentTimeMillis())); - rt = rangeTombstones(cf); + partition = Util.getOnlyPartitionUnfiltered(Util.cmd(cfs, key).fromIncl(5).toIncl(6).build()); + rt = rangeTombstones(partition); assertEquals(1, rt.size()); - cf = cfs.getColumnFamily(QueryFilter.getSliceFilter(dk(key), CFNAME, b(17), b(20), false, Integer.MAX_VALUE, System.currentTimeMillis())); - rt = rangeTombstones(cf); + partition = Util.getOnlyPartitionUnfiltered(Util.cmd(cfs, key).fromIncl(17).toIncl(20).build()); + rt = rangeTombstones(partition); assertEquals(1, rt.size()); - cf = cfs.getColumnFamily(QueryFilter.getSliceFilter(dk(key), CFNAME, b(17), b(18), false, Integer.MAX_VALUE, System.currentTimeMillis())); - rt = rangeTombstones(cf); + partition = Util.getOnlyPartitionUnfiltered(Util.cmd(cfs, key).fromIncl(17).toIncl(18).build()); + rt = rangeTombstones(partition); assertEquals(1, rt.size()); - ColumnSlice[] slices = new ColumnSlice[]{new ColumnSlice( b(1), b(10)), new ColumnSlice( b(16), b(20))}; - IDiskAtomFilter sqf = new SliceQueryFilter(slices, false, Integer.MAX_VALUE); - cf = cfs.getColumnFamily( new QueryFilter(dk(key), CFNAME, sqf, System.currentTimeMillis()) ); - rt = rangeTombstones(cf); + Slices.Builder sb = new Slices.Builder(cfs.getComparator()); + sb.add(Slice.Bound.create(cfs.getComparator(), true, true, 1), Slice.Bound.create(cfs.getComparator(), false, true, 10)); + sb.add(Slice.Bound.create(cfs.getComparator(), true, true, 16), Slice.Bound.create(cfs.getComparator(), false, true, 20)); + + partition = Util.getOnlyPartitionUnfiltered(SinglePartitionSliceCommand.create(cfs.metadata, FBUtilities.nowInSeconds(), Util.dk(key), sb.build())); + rt = rangeTombstones(partition); assertEquals(2, rt.size()); } - private Collection rangeTombstones(ColumnFamily cf) + private Collection rangeTombstones(ArrayBackedPartition partition) { List tombstones = new ArrayList(); - Iterators.addAll(tombstones, cf.deletionInfo().rangeIterator()); + Iterators.addAll(tombstones, partition.deletionInfo().rangeIterator(false)); return tombstones; } @Test - public void testTrackTimesRowTombstone() throws ExecutionException, InterruptedException + public void testTrackTimesPartitionTombstone() throws ExecutionException, InterruptedException { Keyspace ks = Keyspace.open(KSNAME); ColumnFamilyStore cfs = ks.getColumnFamilyStore(CFNAME); cfs.truncateBlocking(); String key = "rt_times"; - Mutation rm = new Mutation(KSNAME, ByteBufferUtil.bytes(key)); - ColumnFamily cf = rm.addOrGet(CFNAME); - long timestamp = System.currentTimeMillis(); - cf.delete(new DeletionInfo(1000, (int)(timestamp/1000))); - rm.apply(); + + int nowInSec = FBUtilities.nowInSeconds(); + new Mutation(PartitionUpdate.fullPartitionDelete(cfs.metadata, Util.dk(key), 1000, nowInSec)).apply(); cfs.forceBlockingFlush(); + SSTableReader sstable = cfs.getSSTables().iterator().next(); - assertTimes(sstable.getSSTableMetadata(), 1000, 1000, (int)(timestamp/1000)); + assertTimes(sstable.getSSTableMetadata(), 1000, 1000, nowInSec); cfs.forceMajorCompaction(); sstable = cfs.getSSTables().iterator().next(); - assertTimes(sstable.getSSTableMetadata(), 1000, 1000, (int)(timestamp/1000)); + assertTimes(sstable.getSSTableMetadata(), 1000, 1000, nowInSec); } @Test - public void testTrackTimesRowTombstoneWithData() throws ExecutionException, InterruptedException + public void testTrackTimesPartitionTombstoneWithData() throws ExecutionException, InterruptedException { Keyspace ks = Keyspace.open(KSNAME); ColumnFamilyStore cfs = ks.getColumnFamilyStore(CFNAME); cfs.truncateBlocking(); String key = "rt_times"; - Mutation rm = new Mutation(KSNAME, ByteBufferUtil.bytes(key)); - add(rm, 5, 999); - rm.apply(); + + UpdateBuilder.create(cfs.metadata, key).withTimestamp(999).newRow(5).add("val", 5).apply(); + key = "rt_times2"; - rm = new Mutation(KSNAME, ByteBufferUtil.bytes(key)); - ColumnFamily cf = rm.addOrGet(CFNAME); - int timestamp = (int)(System.currentTimeMillis()/1000); - cf.delete(new DeletionInfo(1000, timestamp)); - rm.apply(); + int nowInSec = FBUtilities.nowInSeconds(); + new Mutation(PartitionUpdate.fullPartitionDelete(cfs.metadata, Util.dk(key), 1000, nowInSec)).apply(); cfs.forceBlockingFlush(); + SSTableReader sstable = cfs.getSSTables().iterator().next(); assertTimes(sstable.getSSTableMetadata(), 999, 1000, Integer.MAX_VALUE); cfs.forceMajorCompaction(); sstable = cfs.getSSTables().iterator().next(); assertTimes(sstable.getSSTableMetadata(), 999, 1000, Integer.MAX_VALUE); } + @Test public void testTrackTimesRangeTombstone() throws ExecutionException, InterruptedException { @@ -290,17 +273,16 @@ public class RangeTombstoneTest ColumnFamilyStore cfs = ks.getColumnFamilyStore(CFNAME); cfs.truncateBlocking(); String key = "rt_times"; - Mutation rm = new Mutation(KSNAME, ByteBufferUtil.bytes(key)); - ColumnFamily cf = rm.addOrGet(CFNAME); - long timestamp = System.currentTimeMillis(); - cf.delete(new DeletionInfo(b(1), b(2), cfs.getComparator(), 1000, (int)(timestamp/1000))); - rm.apply(); + + int nowInSec = FBUtilities.nowInSeconds(); + new RowUpdateBuilder(cfs.metadata, nowInSec, 1000L, key).addRangeTombstone(1, 2).build().apply(); cfs.forceBlockingFlush(); + SSTableReader sstable = cfs.getSSTables().iterator().next(); - assertTimes(sstable.getSSTableMetadata(), 1000, 1000, (int)(timestamp/1000)); + assertTimes(sstable.getSSTableMetadata(), 1000, 1000, nowInSec); cfs.forceMajorCompaction(); sstable = cfs.getSSTables().iterator().next(); - assertTimes(sstable.getSSTableMetadata(), 1000, 1000, (int)(timestamp/1000)); + assertTimes(sstable.getSSTableMetadata(), 1000, 1000, nowInSec); } @Test @@ -310,15 +292,14 @@ public class RangeTombstoneTest ColumnFamilyStore cfs = ks.getColumnFamilyStore(CFNAME); cfs.truncateBlocking(); String key = "rt_times"; - Mutation rm = new Mutation(KSNAME, ByteBufferUtil.bytes(key)); - add(rm, 5, 999); - rm.apply(); + + UpdateBuilder.create(cfs.metadata, key).withTimestamp(999).newRow(5).add("val", 5).apply(); + key = "rt_times2"; - rm = new Mutation(KSNAME, ByteBufferUtil.bytes(key)); - ColumnFamily cf = rm.addOrGet(CFNAME); - int timestamp = (int)(System.currentTimeMillis()/1000); - cf.delete(new DeletionInfo(b(1), b(2), cfs.getComparator(), 1000, timestamp)); - rm.apply(); + int nowInSec = FBUtilities.nowInSeconds(); + new Mutation(PartitionUpdate.fullPartitionDelete(cfs.metadata, Util.dk(key), 1000, nowInSec)).apply(); + cfs.forceBlockingFlush(); + cfs.forceBlockingFlush(); SSTableReader sstable = cfs.getSSTables().iterator().next(); assertTimes(sstable.getSSTableMetadata(), 999, 1000, Integer.MAX_VALUE); @@ -342,21 +323,19 @@ public class RangeTombstoneTest cfs.metadata.gcGraceSeconds(2); String key = "7810"; - Mutation rm; - rm = new Mutation(KSNAME, ByteBufferUtil.bytes(key)); - for (int i = 10; i < 20; i++) - add(rm, i, 0); - rm.apply(); + + UpdateBuilder builder = UpdateBuilder.create(cfs.metadata, key).withTimestamp(0); + for (int i = 10; i < 20; i ++) + builder.newRow(i).add("val", i); + builder.apply(); cfs.forceBlockingFlush(); - rm = new Mutation(KSNAME, ByteBufferUtil.bytes(key)); - ColumnFamily cf = rm.addOrGet(CFNAME); - cf.delete(new DeletionInfo(b(10),b(11), cfs.getComparator(), 1, 1)); - rm.apply(); + new RowUpdateBuilder(cfs.metadata, 1, key).addRangeTombstone(10, 11).build().apply(); cfs.forceBlockingFlush(); + Thread.sleep(5); cfs.forceMajorCompaction(); - assertEquals(8, Util.getColumnFamily(ks, Util.dk(key), CFNAME).getColumnCount()); + assertEquals(8, Util.getOnlyPartitionUnfiltered(Util.cmd(cfs, key).build()).rowCount()); } @Test @@ -367,16 +346,13 @@ public class RangeTombstoneTest cfs.metadata.gcGraceSeconds(2); String key = "7808_1"; - Mutation rm; - rm = new Mutation(KSNAME, ByteBufferUtil.bytes(key)); + UpdateBuilder builder = UpdateBuilder.create(cfs.metadata, key).withTimestamp(0); for (int i = 0; i < 40; i += 2) - add(rm, i, 0); - rm.apply(); + builder.newRow(i).add("val", i); + builder.apply(); cfs.forceBlockingFlush(); - rm = new Mutation(KSNAME, ByteBufferUtil.bytes(key)); - ColumnFamily cf = rm.addOrGet(CFNAME); - cf.delete(new DeletionInfo(1, 1)); - rm.apply(); + + new Mutation(PartitionUpdate.fullPartitionDelete(cfs.metadata, Util.dk(key), 1, 1)).apply(); cfs.forceBlockingFlush(); Thread.sleep(5); cfs.forceMajorCompaction(); @@ -390,26 +366,20 @@ public class RangeTombstoneTest cfs.metadata.gcGraceSeconds(2); String key = "7808_2"; - Mutation rm; - rm = new Mutation(KSNAME, ByteBufferUtil.bytes(key)); - for (int i = 10; i < 20; i++) - add(rm, i, 0); - rm.apply(); + UpdateBuilder builder = UpdateBuilder.create(cfs.metadata, key).withTimestamp(0); + for (int i = 10; i < 20; i ++) + builder.newRow(i).add("val", i); + builder.apply(); cfs.forceBlockingFlush(); - rm = new Mutation(KSNAME, ByteBufferUtil.bytes(key)); - ColumnFamily cf = rm.addOrGet(CFNAME); - cf.delete(new DeletionInfo(0,0)); - rm.apply(); + new Mutation(PartitionUpdate.fullPartitionDelete(cfs.metadata, Util.dk(key), 0, 0)).apply(); - rm = new Mutation(KSNAME, ByteBufferUtil.bytes(key)); - add(rm, 5, 1); - rm.apply(); + UpdateBuilder.create(cfs.metadata, key).withTimestamp(1).newRow(5).add("val", 5).apply(); cfs.forceBlockingFlush(); Thread.sleep(5); cfs.forceMajorCompaction(); - assertEquals(1, Util.getColumnFamily(ks, Util.dk(key), CFNAME).getColumnCount()); + assertEquals(1, Util.getOnlyPartitionUnfiltered(Util.cmd(cfs, key).build()).rowCount()); } @Test @@ -421,52 +391,41 @@ public class RangeTombstoneTest // Inserting data String key = "k2"; - Mutation rm; - ColumnFamily cf; - rm = new Mutation(KSNAME, ByteBufferUtil.bytes(key)); + UpdateBuilder builder = UpdateBuilder.create(cfs.metadata, key).withTimestamp(0); for (int i = 0; i < 20; i++) - add(rm, i, 0); - rm.applyUnsafe(); + builder.newRow(i).add("val", i); + builder.applyUnsafe(); cfs.forceBlockingFlush(); - rm = new Mutation(KSNAME, ByteBufferUtil.bytes(key)); - cf = rm.addOrGet(CFNAME); - delete(cf, 5, 15, 1); - rm.applyUnsafe(); + new RowUpdateBuilder(cfs.metadata, 1, key).addRangeTombstone(5, 15).build().applyUnsafe(); cfs.forceBlockingFlush(); - rm = new Mutation(KSNAME, ByteBufferUtil.bytes(key)); - cf = rm.addOrGet(CFNAME); - delete(cf, 5, 10, 1); - rm.applyUnsafe(); + new RowUpdateBuilder(cfs.metadata, 1, key).addRangeTombstone(5, 10).build().applyUnsafe(); cfs.forceBlockingFlush(); - rm = new Mutation(KSNAME, ByteBufferUtil.bytes(key)); - cf = rm.addOrGet(CFNAME); - delete(cf, 5, 8, 2); - rm.applyUnsafe(); + new RowUpdateBuilder(cfs.metadata, 2, key).addRangeTombstone(5, 8).build().applyUnsafe(); cfs.forceBlockingFlush(); - cf = cfs.getColumnFamily(QueryFilter.getIdentityFilter(dk(key), CFNAME, System.currentTimeMillis())); + Partition partition = Util.getOnlyPartitionUnfiltered(Util.cmd(cfs, key).build()); for (int i = 0; i < 5; i++) - assertTrue("Cell " + i + " should be live", isLive(cf, cf.getColumn(b(i)))); + assertTrue("Row " + i + " should be live", partition.getRow(new SimpleClustering(bb(i))).hasLiveData(FBUtilities.nowInSeconds())); for (int i = 16; i < 20; i++) - assertTrue("Cell " + i + " should be live", isLive(cf, cf.getColumn(b(i)))); + assertTrue("Row " + i + " should be live", partition.getRow(new SimpleClustering(bb(i))).hasLiveData(FBUtilities.nowInSeconds())); for (int i = 5; i <= 15; i++) - assertTrue("Cell " + i + " shouldn't be live", !isLive(cf, cf.getColumn(b(i)))); + assertTrue("Row " + i + " shouldn't be live", partition.getRow(new SimpleClustering(bb(i))) == null); // Compact everything and re-test CompactionManager.instance.performMaximal(cfs, false); - cf = cfs.getColumnFamily(QueryFilter.getIdentityFilter(dk(key), CFNAME, System.currentTimeMillis())); + partition = Util.getOnlyPartitionUnfiltered(Util.cmd(cfs, key).build()); for (int i = 0; i < 5; i++) - assertTrue("Cell " + i + " should be live", isLive(cf, cf.getColumn(b(i)))); + assertTrue("Row " + i + " should be live", partition.getRow(new SimpleClustering(bb(i))).hasLiveData(FBUtilities.nowInSeconds())); for (int i = 16; i < 20; i++) - assertTrue("Cell " + i + " should be live", isLive(cf, cf.getColumn(b(i)))); + assertTrue("Row " + i + " should be live", partition.getRow(new SimpleClustering(bb(i))).hasLiveData(FBUtilities.nowInSeconds())); for (int i = 5; i <= 15; i++) - assertTrue("Cell " + i + " shouldn't be live", !isLive(cf, cf.getColumn(b(i)))); + assertTrue("Row " + i + " shouldn't be live", partition.getRow(new SimpleClustering(bb(i))) == null); } @Test @@ -477,26 +436,19 @@ public class RangeTombstoneTest // Inserting data String key = "k3"; - Mutation rm; - ColumnFamily cf; - rm = new Mutation(KSNAME, ByteBufferUtil.bytes(key)); - add(rm, 2, 0); - rm.applyUnsafe(); + UpdateBuilder.create(cfs.metadata, key).withTimestamp(0).newRow(2).add("val", 2).applyUnsafe(); cfs.forceBlockingFlush(); - rm = new Mutation(KSNAME, ByteBufferUtil.bytes(key)); - // Deletes everything but without being a row tombstone - delete(rm.addOrGet(CFNAME), 0, 10, 1); - add(rm, 1, 2); - rm.applyUnsafe(); + new RowUpdateBuilder(cfs.metadata, 1, key).addRangeTombstone(0, 10).build().applyUnsafe(); + UpdateBuilder.create(cfs.metadata, key).withTimestamp(2).newRow(1).add("val", 1).applyUnsafe(); cfs.forceBlockingFlush(); // Get the last value of the row - cf = cfs.getColumnFamily(QueryFilter.getSliceFilter(dk(key), CFNAME, Composites.EMPTY, Composites.EMPTY, true, 1, System.currentTimeMillis())); + FilteredPartition partition = Util.getOnlyPartition(Util.cmd(cfs, key).build()); + assertTrue(partition.rowCount() > 0); - assertFalse(cf.isEmpty()); - int last = i(cf.getSortedColumns().iterator().next().name()); + int last = i(partition.unfilteredIterator(ColumnFilter.all(cfs.metadata), Slices.ALL, true).next().clustering().get(0)); assertEquals("Last column should be column 1 since column 2 has been deleted", 1, last); } @@ -518,17 +470,13 @@ public class RangeTombstoneTest cfs.disableAutoCompaction(); cfs.setCompactionStrategyClass(SizeTieredCompactionStrategy.class.getCanonicalName()); - Mutation rm = new Mutation(KSNAME, key); + UpdateBuilder builder = UpdateBuilder.create(cfs.metadata, key).withTimestamp(0); for (int i = 0; i < 10; i += 2) - add(rm, i, 0); - rm.applyUnsafe(); + builder.newRow(i).add("val", i); + builder.applyUnsafe(); cfs.forceBlockingFlush(); - rm = new Mutation(KSNAME, key); - ColumnFamily cf = rm.addOrGet(CFNAME); - for (int i = 0; i < 10; i += 2) - delete(cf, 0, 7, 0); - rm.applyUnsafe(); + new RowUpdateBuilder(cfs.metadata, 0, key).addRangeTombstone(0, 7).build().applyUnsafe(); cfs.forceBlockingFlush(); // there should be 2 sstables @@ -540,21 +488,18 @@ public class RangeTombstoneTest // test the physical structure of the sstable i.e. rt & columns on disk SSTableReader sstable = cfs.getSSTables().iterator().next(); - try(ISSTableScanner scanner = sstable.getScanner()) + try (UnfilteredPartitionIterator scanner = sstable.getScanner()) { - OnDiskAtomIterator iter = scanner.next(); - int cnt = 0; - // after compaction, the first element should be an RT followed by the remaining non-deleted columns - while (iter.hasNext()) + try (UnfilteredRowIterator iter = scanner.next()) { - OnDiskAtom atom = iter.next(); - if (cnt == 0) - assertTrue(atom instanceof RangeTombstone); - if (cnt > 0) - assertTrue(atom instanceof Cell); - cnt++; + // after compaction, we should have a single RT with a single row (the row 8) + Unfiltered u1 = iter.next(); + assertTrue("Expecting open marker, got " + u1.toString(cfs.metadata), u1 instanceof RangeTombstoneMarker); + Unfiltered u2 = iter.next(); + assertTrue("Expecting close marker, got " + u2.toString(cfs.metadata), u2 instanceof RangeTombstoneMarker); + Unfiltered u3 = iter.next(); + assertTrue("Expecting row, got " + u3.toString(cfs.metadata), u3 instanceof Row); } - assertEquals(2, cnt); } } @@ -564,38 +509,29 @@ public class RangeTombstoneTest Keyspace table = Keyspace.open(KSNAME); ColumnFamilyStore cfs = table.getColumnFamilyStore(CFNAME); ByteBuffer key = ByteBufferUtil.bytes("k6"); - ByteBuffer indexedColumnName = ByteBufferUtil.bytes(1); + ByteBuffer indexedColumnName = ByteBufferUtil.bytes("val"); cfs.truncateBlocking(); cfs.disableAutoCompaction(); cfs.setCompactionStrategyClass(SizeTieredCompactionStrategy.class.getCanonicalName()); - if (cfs.indexManager.getIndexForColumn(indexedColumnName) == null) - { - ColumnDefinition cd = new ColumnDefinition(cfs.metadata, indexedColumnName, Int32Type.instance, null, ColumnDefinition.Kind.REGULAR); - cd.setIndex("test_index", IndexType.CUSTOM, ImmutableMap.of(SecondaryIndex.CUSTOM_INDEX_OPTION_NAME, TestIndex.class.getName())); - Future rebuild = cfs.indexManager.addIndexedColumn(cd); - // If rebuild there is, wait for the rebuild to finish so it doesn't race with the following insertions - if (rebuild != null) - rebuild.get(); - } - TestIndex index = ((TestIndex)cfs.indexManager.getIndexForColumn(indexedColumnName)); + ColumnDefinition cd = cfs.metadata.getColumnDefinition(indexedColumnName).copy(); + cd.setIndex("test_index", IndexType.CUSTOM, ImmutableMap.of(SecondaryIndex.CUSTOM_INDEX_OPTION_NAME, TestIndex.class.getName())); + Future rebuild = cfs.indexManager.addIndexedColumn(cd); + // If rebuild there is, wait for the rebuild to finish so it doesn't race with the following insertions + if (rebuild != null) + rebuild.get(); + + TestIndex index = ((TestIndex)cfs.indexManager.getIndexForColumn(cd)); index.resetCounts(); - Mutation rm = new Mutation(KSNAME, key); - add(rm, 1, 0); - rm.applyUnsafe(); + UpdateBuilder.create(cfs.metadata, key).withTimestamp(0).newRow(1).add("val", 1).applyUnsafe(); // add a RT which hides the column we just inserted - rm = new Mutation(KSNAME, key); - ColumnFamily cf = rm.addOrGet(CFNAME); - delete(cf, 0, 1, 1); - rm.applyUnsafe(); + new RowUpdateBuilder(cfs.metadata, 1, key).addRangeTombstone(0, 1).build().applyUnsafe(); // now re-insert that column - rm = new Mutation(KSNAME, key); - add(rm, 1, 2); - rm.applyUnsafe(); + UpdateBuilder.create(cfs.metadata, key).withTimestamp(2).newRow(1).add("val", 1).applyUnsafe(); cfs.forceBlockingFlush(); @@ -610,78 +546,50 @@ public class RangeTombstoneTest Keyspace table = Keyspace.open(KSNAME); ColumnFamilyStore cfs = table.getColumnFamilyStore(CFNAME); ByteBuffer key = ByteBufferUtil.bytes("k5"); - ByteBuffer indexedColumnName = ByteBufferUtil.bytes(1); + ByteBuffer indexedColumnName = ByteBufferUtil.bytes("val"); cfs.truncateBlocking(); cfs.disableAutoCompaction(); cfs.setCompactionStrategyClass(SizeTieredCompactionStrategy.class.getCanonicalName()); - if (cfs.indexManager.getIndexForColumn(indexedColumnName) == null) - { - ColumnDefinition cd = ColumnDefinition.regularDef(cfs.metadata, indexedColumnName, cfs.getComparator().asAbstractType(), 0) - .setIndex("test_index", IndexType.CUSTOM, ImmutableMap.of(SecondaryIndex.CUSTOM_INDEX_OPTION_NAME, TestIndex.class.getName())); - Future rebuild = cfs.indexManager.addIndexedColumn(cd); - // If rebuild there is, wait for the rebuild to finish so it doesn't race with the following insertions - if (rebuild != null) - rebuild.get(); - } - TestIndex index = ((TestIndex)cfs.indexManager.getIndexForColumn(indexedColumnName)); + ColumnDefinition cd = cfs.metadata.getColumnDefinition(indexedColumnName).copy(); + cd.setIndex("test_index", IndexType.CUSTOM, ImmutableMap.of(SecondaryIndex.CUSTOM_INDEX_OPTION_NAME, TestIndex.class.getName())); + Future rebuild = cfs.indexManager.addIndexedColumn(cd); + // If rebuild there is, wait for the rebuild to finish so it doesn't race with the following insertions + if (rebuild != null) + rebuild.get(); + + TestIndex index = ((TestIndex)cfs.indexManager.getIndexForColumn(cd)); index.resetCounts(); - Mutation rm = new Mutation(KSNAME, key); + UpdateBuilder builder = UpdateBuilder.create(cfs.metadata, key).withTimestamp(0); for (int i = 0; i < 10; i++) - add(rm, i, 0); - rm.applyUnsafe(); + builder.newRow(i).add("val", i); + builder.applyUnsafe(); cfs.forceBlockingFlush(); - rm = new Mutation(KSNAME, key); - ColumnFamily cf = rm.addOrGet(CFNAME); - for (int i = 0; i < 10; i += 2) - delete(cf, 0, 7, 0); - rm.applyUnsafe(); + new RowUpdateBuilder(cfs.metadata, 0, key).addRangeTombstone(0, 7).build().applyUnsafe(); cfs.forceBlockingFlush(); - // We should have indexed 1 column - assertEquals(1, index.inserts.size()); + assertEquals(10, index.inserts.size()); CompactionManager.instance.performMaximal(cfs, false); // compacted down to single sstable assertEquals(1, cfs.getSSTables().size()); - // verify that the 1 indexed column was removed from the index - assertEquals(1, index.deletes.size()); - assertEquals(index.deletes.get(0), index.inserts.get(0)); + assertEquals(10, index.deletes.size()); } - private static boolean isLive(ColumnFamily cf, Cell c) + private static ByteBuffer bb(int i) { - return c != null && c.isLive() && !cf.deletionInfo().isDeleted(c); + return ByteBufferUtil.bytes(i); } - private static CellName b(int i) + private static int i(ByteBuffer bb) { - return CellNames.simpleDense(ByteBufferUtil.bytes(i)); - } - - private static int i(CellName i) - { - return ByteBufferUtil.toInt(i.toByteBuffer()); - } - - private static void add(Mutation rm, int value, long timestamp) - { - rm.add(CFNAME, b(value), ByteBufferUtil.bytes(value), timestamp); - } - - private static void delete(ColumnFamily cf, int from, int to, long timestamp) - { - cf.delete(new DeletionInfo(b(from), - b(to), - cf.getComparator(), - timestamp, - (int)(System.currentTimeMillis() / 1000))); + return ByteBufferUtil.toInt(bb); } public static class TestIndex extends PerColumnSecondaryIndex @@ -697,22 +605,22 @@ public class RangeTombstoneTest updates.clear(); } - public void delete(ByteBuffer rowKey, Cell col, OpOrder.Group opGroup) + public void delete(ByteBuffer rowKey, Clustering clustering, Cell cell, OpOrder.Group opGroup, int nowInSec) { - deletes.add(col); + deletes.add(cell); } @Override - public void deleteForCleanup(ByteBuffer rowKey, Cell col, OpOrder.Group opGroup) {} + public void deleteForCleanup(ByteBuffer rowKey, Clustering clustering, Cell cell, OpOrder.Group opGroup, int nowInSec) {} - public void insert(ByteBuffer rowKey, Cell col, OpOrder.Group opGroup) + public void insert(ByteBuffer rowKey, Clustering clustering, Cell cell, OpOrder.Group opGroup) { - inserts.add(col); + inserts.add(cell); } - public void update(ByteBuffer rowKey, Cell oldCol, Cell col, OpOrder.Group opGroup) + public void update(ByteBuffer rowKey, Clustering clustering, Cell oldCell, Cell cell, OpOrder.Group opGroup, int nowInSec) { - updates.add(col); + updates.add(cell); } public void init(){} @@ -723,7 +631,7 @@ public class RangeTombstoneTest public String getIndexName(){ return "TestIndex";} - protected SecondaryIndexSearcher createSecondaryIndexSearcher(Set columns){ return null; } + protected SecondaryIndexSearcher createSecondaryIndexSearcher(Set columns) { return null; }; public void forceBlockingFlush(){} @@ -733,12 +641,17 @@ public class RangeTombstoneTest public void invalidate(){} + public void validate(DecoratedKey key) {} + public void validate(ByteBuffer value, CellPath path) {} + public void validate(Clustering clustering) {} + public void truncateBlocking(long truncatedAt) { } - public boolean indexes(CellName name) { return name.toByteBuffer().equals(ByteBufferUtil.bytes(1)); } + public boolean indexes(ColumnDefinition name) { return true; } @Override - public long estimateResultRows() { + public long estimateResultRows() + { return 0; } } diff --git a/test/unit/org/apache/cassandra/db/ReadMessageTest.java b/test/unit/org/apache/cassandra/db/ReadMessageTest.java index 34f25a175f..b7e19f81bf 100644 --- a/test/unit/org/apache/cassandra/db/ReadMessageTest.java +++ b/test/unit/org/apache/cassandra/db/ReadMessageTest.java @@ -21,9 +21,6 @@ package org.apache.cassandra.db; import static org.junit.Assert.*; import java.io.*; -import java.nio.ByteBuffer; -import java.util.SortedSet; -import java.util.TreeSet; import com.google.common.base.Predicate; import org.junit.BeforeClass; @@ -31,109 +28,183 @@ import org.junit.Test; import org.apache.cassandra.SchemaLoader; import org.apache.cassandra.Util; +import org.apache.cassandra.config.CFMetaData; +import org.apache.cassandra.config.ColumnDefinition; +import org.apache.cassandra.config.DatabaseDescriptor; import org.apache.cassandra.config.KSMetaData; import org.apache.cassandra.db.commitlog.CommitLogTestReplayer; -import org.apache.cassandra.db.composites.*; -import org.apache.cassandra.db.filter.NamesQueryFilter; -import org.apache.cassandra.db.filter.SliceQueryFilter; +import org.apache.cassandra.db.partitions.PartitionUpdate; +import org.apache.cassandra.db.rows.Row; +import org.apache.cassandra.db.rows.RowIterator; +import org.apache.cassandra.db.marshal.AsciiType; +import org.apache.cassandra.db.marshal.BytesType; +import org.apache.cassandra.db.partitions.PartitionIterator; +import org.apache.cassandra.db.partitions.FilteredPartition; import org.apache.cassandra.exceptions.ConfigurationException; +import org.apache.cassandra.io.IVersionedSerializer; import org.apache.cassandra.io.util.DataOutputBuffer; import org.apache.cassandra.locator.SimpleStrategy; import org.apache.cassandra.net.MessagingService; import org.apache.cassandra.utils.ByteBufferUtil; import org.apache.cassandra.utils.FBUtilities; +import static org.junit.Assert.*; public class ReadMessageTest { private static final String KEYSPACE1 = "ReadMessageTest1"; private static final String KEYSPACENOCOMMIT = "ReadMessageTest_NoCommit"; private static final String CF = "Standard1"; + private static final String CF_FOR_READ_TEST = "Standard2"; + private static final String CF_FOR_COMMIT_TEST = "Standard3"; @BeforeClass public static void defineSchema() throws ConfigurationException { + CFMetaData cfForReadMetadata = CFMetaData.Builder.create(KEYSPACE1, CF_FOR_READ_TEST) + .addPartitionKey("key", BytesType.instance) + .addClusteringColumn("col1", AsciiType.instance) + .addClusteringColumn("col2", AsciiType.instance) + .addRegularColumn("a", AsciiType.instance) + .addRegularColumn("b", AsciiType.instance).build(); + + CFMetaData cfForCommitMetadata1 = CFMetaData.Builder.create(KEYSPACE1, CF_FOR_COMMIT_TEST) + .addPartitionKey("key", BytesType.instance) + .addClusteringColumn("name", AsciiType.instance) + .addRegularColumn("commit1", AsciiType.instance).build(); + + CFMetaData cfForCommitMetadata2 = CFMetaData.Builder.create(KEYSPACENOCOMMIT, CF_FOR_COMMIT_TEST) + .addPartitionKey("key", BytesType.instance) + .addClusteringColumn("name", AsciiType.instance) + .addRegularColumn("commit2", AsciiType.instance).build(); + SchemaLoader.prepareServer(); SchemaLoader.createKeyspace(KEYSPACE1, SimpleStrategy.class, KSMetaData.optsWithRF(1), - SchemaLoader.standardCFMD(KEYSPACE1, CF)); + SchemaLoader.standardCFMD(KEYSPACE1, CF), + cfForReadMetadata, + cfForCommitMetadata1); SchemaLoader.createKeyspace(KEYSPACENOCOMMIT, false, true, SimpleStrategy.class, KSMetaData.optsWithRF(1), - SchemaLoader.standardCFMD(KEYSPACENOCOMMIT, CF)); + SchemaLoader.standardCFMD(KEYSPACENOCOMMIT, CF), + cfForCommitMetadata2); } @Test public void testMakeReadMessage() throws IOException { - CellNameType type = Keyspace.open(KEYSPACE1).getColumnFamilyStore("Standard1").getComparator(); - - SortedSet colList = new TreeSet(type); - colList.add(Util.cellname("col1")); - colList.add(Util.cellname("col2")); - + ColumnFamilyStore cfs = Keyspace.open(KEYSPACE1).getColumnFamilyStore(CF_FOR_READ_TEST); ReadCommand rm, rm2; - DecoratedKey dk = Util.dk("row1"); - long ts = System.currentTimeMillis(); - rm = new SliceByNamesReadCommand(KEYSPACE1, dk.getKey(), "Standard1", ts, new NamesQueryFilter(colList)); + rm = Util.cmd(cfs, Util.dk("key1")) + .includeRow("col1", "col2") + .build(); rm2 = serializeAndDeserializeReadMessage(rm); - assert rm2.toString().equals(rm.toString()); + assertEquals(rm.toString(), rm2.toString()); - rm = new SliceFromReadCommand(KEYSPACE1, dk.getKey(), "Standard1", ts, new SliceQueryFilter(Composites.EMPTY, Composites.EMPTY, true, 2)); + rm = Util.cmd(cfs, Util.dk("key1")) + .includeRow("col1", "col2") + .reverse() + .build(); rm2 = serializeAndDeserializeReadMessage(rm); - assert rm2.toString().equals(rm.toString()); + assertEquals(rm.toString(), rm2.toString()); - rm = new SliceFromReadCommand(KEYSPACE1, dk.getKey(), "Standard1", ts, new SliceQueryFilter(Util.cellname("a"), Util.cellname("z"), true, 5)); + rm = Util.cmd(cfs) + .build(); rm2 = serializeAndDeserializeReadMessage(rm); - assert rm2.toString().equals(rm.toString()); + assertEquals(rm.toString(), rm2.toString()); + + rm = Util.cmd(cfs) + .fromKeyIncl(ByteBufferUtil.bytes("key1")) + .toKeyIncl(ByteBufferUtil.bytes("key2")) + .build(); + rm2 = serializeAndDeserializeReadMessage(rm); + assertEquals(rm.toString(), rm2.toString()); + + rm = Util.cmd(cfs) + .columns("a") + .build(); + rm2 = serializeAndDeserializeReadMessage(rm); + assertEquals(rm.toString(), rm2.toString()); + + rm = Util.cmd(cfs) + .includeRow("col1", "col2") + .columns("a") + .build(); + rm2 = serializeAndDeserializeReadMessage(rm); + assertEquals(rm.toString(), rm2.toString()); + + rm = Util.cmd(cfs) + .fromKeyIncl(ByteBufferUtil.bytes("key1")) + .includeRow("col1", "col2") + .columns("a") + .build(); + rm2 = serializeAndDeserializeReadMessage(rm); + assertEquals(rm.toString(), rm2.toString()); } private ReadCommand serializeAndDeserializeReadMessage(ReadCommand rm) throws IOException { - ReadCommandSerializer rms = ReadCommand.serializer; + IVersionedSerializer rms = ReadCommand.serializer; DataOutputBuffer out = new DataOutputBuffer(); ByteArrayInputStream bis; rms.serialize(rm, out, MessagingService.current_version); + bis = new ByteArrayInputStream(out.getData(), 0, out.getLength()); return rms.deserialize(new DataInputStream(bis), MessagingService.current_version); } + @Test public void testGetColumn() { - Keyspace keyspace = Keyspace.open(KEYSPACE1); - CellNameType type = keyspace.getColumnFamilyStore("Standard1").getComparator(); - Mutation rm; - DecoratedKey dk = Util.dk("key1"); + ColumnFamilyStore cfs = Keyspace.open(KEYSPACE1).getColumnFamilyStore(CF); - // add data - rm = new Mutation(KEYSPACE1, dk.getKey()); - rm.add("Standard1", Util.cellname("Column1"), ByteBufferUtil.bytes("abcd"), 0); - rm.apply(); + new RowUpdateBuilder(cfs.metadata, 0, ByteBufferUtil.bytes("key1")) + .clustering("Column1") + .add("val", ByteBufferUtil.bytes("abcd")) + .build() + .apply(); - ReadCommand command = new SliceByNamesReadCommand(KEYSPACE1, dk.getKey(), "Standard1", System.currentTimeMillis(), new NamesQueryFilter(FBUtilities.singleton(Util.cellname("Column1"), type))); - Row row = command.getRow(keyspace); - Cell col = row.cf.getColumn(Util.cellname("Column1")); - assertEquals(col.value(), ByteBuffer.wrap("abcd".getBytes())); + ColumnDefinition col = cfs.metadata.getColumnDefinition(ByteBufferUtil.bytes("val")); + int found = 0; + for (FilteredPartition partition : Util.getAll(Util.cmd(cfs).build())) + { + for (Row r : partition) + { + if (r.getCell(col).value().equals(ByteBufferUtil.bytes("abcd"))) + ++found; + } + } + assertEquals(1, found); } @Test public void testNoCommitLog() throws Exception { - Mutation rm = new Mutation(KEYSPACE1, ByteBufferUtil.bytes("row")); - rm.add("Standard1", Util.cellname("commit1"), ByteBufferUtil.bytes("abcd"), 0); - rm.apply(); + ColumnFamilyStore cfs = Keyspace.open(KEYSPACE1).getColumnFamilyStore(CF_FOR_COMMIT_TEST); - rm = new Mutation(KEYSPACENOCOMMIT, ByteBufferUtil.bytes("row")); - rm.add("Standard1", Util.cellname("commit2"), ByteBufferUtil.bytes("abcd"), 0); - rm.apply(); + ColumnFamilyStore cfsnocommit = Keyspace.open(KEYSPACENOCOMMIT).getColumnFamilyStore(CF_FOR_COMMIT_TEST); - Checker checker = new Checker(); + new RowUpdateBuilder(cfs.metadata, 0, ByteBufferUtil.bytes("row")) + .clustering("c") + .add("commit1", ByteBufferUtil.bytes("abcd")) + .build() + .apply(); + + new RowUpdateBuilder(cfsnocommit.metadata, 0, ByteBufferUtil.bytes("row")) + .clustering("c") + .add("commit2", ByteBufferUtil.bytes("abcd")) + .build() + .apply(); + + Checker checker = new Checker(cfs.metadata.getColumnDefinition(ByteBufferUtil.bytes("commit1")), + cfsnocommit.metadata.getColumnDefinition(ByteBufferUtil.bytes("commit2"))); CommitLogTestReplayer.examineCommitLog(checker); assertTrue(checker.commitLogMessageFound); @@ -142,17 +213,30 @@ public class ReadMessageTest static class Checker implements Predicate { + private final ColumnDefinition withCommit; + private final ColumnDefinition withoutCommit; + boolean commitLogMessageFound = false; boolean noCommitLogMessageFound = false; + public Checker(ColumnDefinition withCommit, ColumnDefinition withoutCommit) + { + this.withCommit = withCommit; + this.withoutCommit = withoutCommit; + } + public boolean apply(Mutation mutation) { - for (ColumnFamily cf : mutation.getColumnFamilies()) + for (PartitionUpdate upd : mutation.getPartitionUpdates()) { - if (cf.getColumn(Util.cellname("commit1")) != null) - commitLogMessageFound = true; - if (cf.getColumn(Util.cellname("commit2")) != null) - noCommitLogMessageFound = true; + Row r = upd.getRow(new SimpleClustering(ByteBufferUtil.bytes("c"))); + if (r != null) + { + if (r.getCell(withCommit) != null) + commitLogMessageFound = true; + if (r.getCell(withoutCommit) != null) + noCommitLogMessageFound = true; + } } return true; } diff --git a/test/unit/org/apache/cassandra/db/RecoveryManager2Test.java b/test/unit/org/apache/cassandra/db/RecoveryManagerFlushedTest.java similarity index 79% rename from test/unit/org/apache/cassandra/db/RecoveryManager2Test.java rename to test/unit/org/apache/cassandra/db/RecoveryManagerFlushedTest.java index 13c3452ab0..b5f05f4401 100644 --- a/test/unit/org/apache/cassandra/db/RecoveryManager2Test.java +++ b/test/unit/org/apache/cassandra/db/RecoveryManagerFlushedTest.java @@ -27,19 +27,17 @@ import org.junit.Test; import org.slf4j.Logger; import org.slf4j.LoggerFactory; -import static org.apache.cassandra.Util.column; import org.apache.cassandra.SchemaLoader; import org.apache.cassandra.config.KSMetaData; import org.apache.cassandra.db.compaction.CompactionManager; import org.apache.cassandra.db.commitlog.CommitLog; import org.apache.cassandra.exceptions.ConfigurationException; import org.apache.cassandra.locator.SimpleStrategy; -import org.apache.cassandra.utils.ByteBufferUtil; import org.apache.cassandra.utils.FBUtilities; -public class RecoveryManager2Test +public class RecoveryManagerFlushedTest { - private static Logger logger = LoggerFactory.getLogger(RecoveryManager2Test.class); + private static Logger logger = LoggerFactory.getLogger(RecoveryManagerFlushedTest.class); private static final String KEYSPACE1 = "RecoveryManager2Test"; private static final String CF_STANDARD1 = "Standard1"; @@ -50,10 +48,10 @@ public class RecoveryManager2Test { SchemaLoader.prepareServer(); SchemaLoader.createKeyspace(KEYSPACE1, - SimpleStrategy.class, - KSMetaData.optsWithRF(1), - SchemaLoader.standardCFMD(KEYSPACE1, CF_STANDARD1), - SchemaLoader.standardCFMD(KEYSPACE1, CF_STANDARD2)); + SimpleStrategy.class, + KSMetaData.optsWithRF(1), + SchemaLoader.standardCFMD(KEYSPACE1, CF_STANDARD1), + SchemaLoader.standardCFMD(KEYSPACE1, CF_STANDARD2)); } @Test @@ -86,11 +84,14 @@ public class RecoveryManager2Test assert replayed == 1 : "Expecting only 1 replayed mutation, got " + replayed; } - private void insertRow(String cfname, String key) + private void insertRow(String cfname, String key) { - ColumnFamily cf = ArrayBackedSortedColumns.factory.create(KEYSPACE1, cfname); - cf.addColumn(column("col1", "val1", 1L)); - Mutation rm = new Mutation(KEYSPACE1, ByteBufferUtil.bytes(key), cf); - rm.apply(); + Keyspace keyspace = Keyspace.open(KEYSPACE1); + ColumnFamilyStore cfs = keyspace.getColumnFamilyStore(cfname); + new RowUpdateBuilder(cfs.metadata, 0, key) + .clustering("c") + .add("val", "val1") + .build() + .apply(); } } diff --git a/test/unit/org/apache/cassandra/db/RecoveryManager3Test.java b/test/unit/org/apache/cassandra/db/RecoveryManagerMissingHeaderTest.java similarity index 71% rename from test/unit/org/apache/cassandra/db/RecoveryManager3Test.java rename to test/unit/org/apache/cassandra/db/RecoveryManagerMissingHeaderTest.java index a94d94d8ff..0d85373a10 100644 --- a/test/unit/org/apache/cassandra/db/RecoveryManager3Test.java +++ b/test/unit/org/apache/cassandra/db/RecoveryManagerMissingHeaderTest.java @@ -24,6 +24,7 @@ package org.apache.cassandra.db; import java.io.File; import java.io.IOException; +import org.junit.Assert; import org.junit.BeforeClass; import org.junit.Test; @@ -31,15 +32,14 @@ import org.apache.cassandra.SchemaLoader; import org.apache.cassandra.Util; import org.apache.cassandra.config.DatabaseDescriptor; import org.apache.cassandra.config.KSMetaData; +import org.apache.cassandra.db.rows.AbstractUnfilteredRowIterator; +import org.apache.cassandra.db.rows.UnfilteredRowIterator; import org.apache.cassandra.db.commitlog.CommitLog; import org.apache.cassandra.exceptions.ConfigurationException; import org.apache.cassandra.io.util.FileUtils; import org.apache.cassandra.locator.SimpleStrategy; -import static org.apache.cassandra.Util.column; -import static org.apache.cassandra.db.KeyspaceTest.assertColumns; - -public class RecoveryManager3Test +public class RecoveryManagerMissingHeaderTest { private static final String KEYSPACE1 = "RecoveryManager3Test1"; private static final String CF_STANDARD1 = "Standard1"; @@ -67,19 +67,14 @@ public class RecoveryManager3Test Keyspace keyspace1 = Keyspace.open(KEYSPACE1); Keyspace keyspace2 = Keyspace.open(KEYSPACE2); - Mutation rm; DecoratedKey dk = Util.dk("keymulti"); - ColumnFamily cf; + UnfilteredRowIterator upd1 = Util.apply(new RowUpdateBuilder(keyspace1.getColumnFamilyStore(CF_STANDARD1).metadata, 1L, 0, "keymulti") + .clustering("col1").add("val", "1") + .build()); - cf = ArrayBackedSortedColumns.factory.create(KEYSPACE1, "Standard1"); - cf.addColumn(column("col1", "val1", 1L)); - rm = new Mutation(KEYSPACE1, dk.getKey(), cf); - rm.apply(); - - cf = ArrayBackedSortedColumns.factory.create(KEYSPACE2, "Standard3"); - cf.addColumn(column("col2", "val2", 1L)); - rm = new Mutation(KEYSPACE2, dk.getKey(), cf); - rm.apply(); + UnfilteredRowIterator upd2 = Util.apply(new RowUpdateBuilder(keyspace2.getColumnFamilyStore(CF_STANDARD3).metadata, 1L, 0, "keymulti") + .clustering("col1").add("val", "1") + .build()); keyspace1.getColumnFamilyStore("Standard1").clearUnsafe(); keyspace2.getColumnFamilyStore("Standard3").clearUnsafe(); @@ -91,9 +86,9 @@ public class RecoveryManager3Test FileUtils.deleteWithConfirm(file); } - CommitLog.instance.resetUnsafe(false); // disassociate segments from live CL + CommitLog.instance.resetUnsafe(false); - assertColumns(Util.getColumnFamily(keyspace1, dk, "Standard1"), "col1"); - assertColumns(Util.getColumnFamily(keyspace2, dk, "Standard3"), "col2"); + Assert.assertTrue(AbstractUnfilteredRowIterator.equal(upd1, Util.getOnlyPartitionUnfiltered(Util.cmd(keyspace1.getColumnFamilyStore(CF_STANDARD1), dk).build()).unfilteredIterator())); + Assert.assertTrue(AbstractUnfilteredRowIterator.equal(upd2, Util.getOnlyPartitionUnfiltered(Util.cmd(keyspace2.getColumnFamilyStore(CF_STANDARD3), dk).build()).unfilteredIterator())); } } diff --git a/test/unit/org/apache/cassandra/db/RecoveryManagerTest.java b/test/unit/org/apache/cassandra/db/RecoveryManagerTest.java index c9abe0d68b..f9d96b62ae 100644 --- a/test/unit/org/apache/cassandra/db/RecoveryManagerTest.java +++ b/test/unit/org/apache/cassandra/db/RecoveryManagerTest.java @@ -22,30 +22,39 @@ import java.io.IOException; import java.util.Date; import java.util.concurrent.TimeUnit; +import com.google.common.collect.Iterators; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + import org.apache.cassandra.OrderedJUnit4ClassRunner; import org.apache.cassandra.Util; -import org.apache.cassandra.config.CFMetaData; +import org.apache.cassandra.config.ColumnDefinition; import org.apache.cassandra.config.KSMetaData; -import org.apache.cassandra.db.marshal.BytesType; -import org.apache.cassandra.db.marshal.CounterColumnType; +import org.apache.cassandra.db.rows.*; +import org.apache.cassandra.db.context.CounterContext; +import org.apache.cassandra.db.partitions.PartitionIterator; import org.apache.cassandra.exceptions.ConfigurationException; import org.apache.cassandra.locator.SimpleStrategy; + import org.junit.Assert; +import org.junit.Before; import org.junit.BeforeClass; import org.junit.Test; import org.junit.runner.RunWith; +import static org.junit.Assert.assertEquals; + import org.apache.cassandra.SchemaLoader; import org.apache.cassandra.db.commitlog.CommitLog; import org.apache.cassandra.db.commitlog.CommitLogArchiver; - -import static org.apache.cassandra.Util.column; -import static org.apache.cassandra.db.KeyspaceTest.assertColumns; -import static org.apache.cassandra.Util.cellname; +import org.apache.cassandra.utils.ByteBufferUtil; +import org.apache.cassandra.utils.FBUtilities; @RunWith(OrderedJUnit4ClassRunner.class) public class RecoveryManagerTest { + private static Logger logger = LoggerFactory.getLogger(RecoveryManagerTest.class); + private static final String KEYSPACE1 = "RecoveryManagerTest1"; private static final String CF_STANDARD1 = "Standard1"; private static final String CF_COUNTER1 = "Counter1"; @@ -61,13 +70,21 @@ public class RecoveryManagerTest SimpleStrategy.class, KSMetaData.optsWithRF(1), SchemaLoader.standardCFMD(KEYSPACE1, CF_STANDARD1), - SchemaLoader.standardCFMD(KEYSPACE1, CF_COUNTER1).defaultValidator(CounterColumnType.instance)); + SchemaLoader.counterCFMD(KEYSPACE1, CF_COUNTER1)); SchemaLoader.createKeyspace(KEYSPACE2, SimpleStrategy.class, KSMetaData.optsWithRF(1), SchemaLoader.standardCFMD(KEYSPACE2, CF_STANDARD3)); } + @Before + public void clearData() + { + Keyspace.open(KEYSPACE1).getColumnFamilyStore(CF_STANDARD1).truncateBlocking(); + Keyspace.open(KEYSPACE1).getColumnFamilyStore(CF_COUNTER1).truncateBlocking(); + Keyspace.open(KEYSPACE2).getColumnFamilyStore(CF_STANDARD3).truncateBlocking(); + } + @Test public void testNothingToRecover() throws IOException { @@ -81,27 +98,22 @@ public class RecoveryManagerTest Keyspace keyspace1 = Keyspace.open(KEYSPACE1); Keyspace keyspace2 = Keyspace.open(KEYSPACE2); - Mutation rm; - DecoratedKey dk = Util.dk("keymulti"); - ColumnFamily cf; + UnfilteredRowIterator upd1 = Util.apply(new RowUpdateBuilder(keyspace1.getColumnFamilyStore(CF_STANDARD1).metadata, 1L, 0, "keymulti") + .clustering("col1").add("val", "1") + .build()); - cf = ArrayBackedSortedColumns.factory.create(KEYSPACE1, "Standard1"); - cf.addColumn(column("col1", "val1", 1L)); - rm = new Mutation(KEYSPACE1, dk.getKey(), cf); - rm.apply(); - - cf = ArrayBackedSortedColumns.factory.create(KEYSPACE2, "Standard3"); - cf.addColumn(column("col2", "val2", 1L)); - rm = new Mutation(KEYSPACE2, dk.getKey(), cf); - rm.apply(); + UnfilteredRowIterator upd2 = Util.apply(new RowUpdateBuilder(keyspace2.getColumnFamilyStore(CF_STANDARD3).metadata, 1L, 0, "keymulti") + .clustering("col2").add("val", "1") + .build()); keyspace1.getColumnFamilyStore("Standard1").clearUnsafe(); keyspace2.getColumnFamilyStore("Standard3").clearUnsafe(); - CommitLog.instance.resetUnsafe(false); // disassociate segments from live CL + CommitLog.instance.resetUnsafe(false); - assertColumns(Util.getColumnFamily(keyspace1, dk, "Standard1"), "col1"); - assertColumns(Util.getColumnFamily(keyspace2, dk, "Standard3"), "col2"); + DecoratedKey dk = Util.dk("keymulti"); + Assert.assertTrue(AbstractUnfilteredRowIterator.equal(upd1, Util.getOnlyPartitionUnfiltered(Util.cmd(keyspace1.getColumnFamilyStore(CF_STANDARD1), dk).build()).unfilteredIterator())); + Assert.assertTrue(AbstractUnfilteredRowIterator.equal(upd2, Util.getOnlyPartitionUnfiltered(Util.cmd(keyspace2.getColumnFamilyStore(CF_STANDARD3), dk).build()).unfilteredIterator())); } @Test @@ -109,89 +121,84 @@ public class RecoveryManagerTest { CommitLog.instance.resetUnsafe(true); Keyspace keyspace1 = Keyspace.open(KEYSPACE1); - - Mutation rm; - DecoratedKey dk = Util.dk("key"); - ColumnFamily cf; + ColumnFamilyStore cfs = keyspace1.getColumnFamilyStore(CF_COUNTER1); for (int i = 0; i < 10; ++i) { - cf = ArrayBackedSortedColumns.factory.create(KEYSPACE1, "Counter1"); - cf.addColumn(BufferCounterCell.createLocal(cellname("col"), 1L, 1L, Long.MIN_VALUE)); - rm = new Mutation(KEYSPACE1, dk.getKey(), cf); - rm.apply(); + new CounterMutation(new RowUpdateBuilder(cfs.metadata, 1L, 0, "key") + .clustering("cc").add("val", CounterContext.instance().createLocal(1L)) + .build(), ConsistencyLevel.ALL).apply(); } keyspace1.getColumnFamilyStore("Counter1").clearUnsafe(); - CommitLog.instance.resetUnsafe(false); // disassociate segments from live CL + int replayed = CommitLog.instance.resetUnsafe(false); - cf = Util.getColumnFamily(keyspace1, dk, "Counter1"); - - assert cf.getColumnCount() == 1; - Cell c = cf.getColumn(cellname("col")); - - assert c != null; - assert ((CounterCell)c).total() == 10L; + ColumnDefinition counterCol = cfs.metadata.getColumnDefinition(ByteBufferUtil.bytes("val")); + Row row = Util.getOnlyRow(Util.cmd(cfs).includeRow("cc").columns("val").build()); + assertEquals(10L, CounterContext.instance().total(row.getCell(counterCol).value())); } @Test public void testRecoverPIT() throws Exception { + ColumnFamilyStore cfs = Keyspace.open(KEYSPACE1).getColumnFamilyStore(CF_STANDARD1); CommitLog.instance.resetUnsafe(true); Date date = CommitLogArchiver.format.parse("2112:12:12 12:12:12"); long timeMS = date.getTime() - 5000; Keyspace keyspace1 = Keyspace.open(KEYSPACE1); - DecoratedKey dk = Util.dk("dkey"); for (int i = 0; i < 10; ++i) { long ts = TimeUnit.MILLISECONDS.toMicros(timeMS + (i * 1000)); - ColumnFamily cf = ArrayBackedSortedColumns.factory.create(KEYSPACE1, "Standard1"); - cf.addColumn(column("name-" + i, "value", ts)); - Mutation rm = new Mutation(KEYSPACE1, dk.getKey(), cf); - rm.apply(); + new RowUpdateBuilder(cfs.metadata, ts, "name-" + i) + .clustering("cc") + .add("val", Integer.toString(i)) + .build() + .apply(); } + + // Sanity check row count prior to clear and replay + assertEquals(10, Util.getAll(Util.cmd(cfs).build()).size()); + keyspace1.getColumnFamilyStore("Standard1").clearUnsafe(); - CommitLog.instance.resetUnsafe(false); // disassociate segments from live CL + CommitLog.instance.resetUnsafe(false); - ColumnFamily cf = Util.getColumnFamily(keyspace1, dk, "Standard1"); - Assert.assertEquals(6, cf.getColumnCount()); + assertEquals(6, Util.getAll(Util.cmd(cfs).build()).size()); } - @Test public void testRecoverPITUnordered() throws Exception { + ColumnFamilyStore cfs = Keyspace.open(KEYSPACE1).getColumnFamilyStore(CF_STANDARD1); CommitLog.instance.resetUnsafe(true); Date date = CommitLogArchiver.format.parse("2112:12:12 12:12:12"); long timeMS = date.getTime(); Keyspace keyspace1 = Keyspace.open(KEYSPACE1); - DecoratedKey dk = Util.dk("dkey"); // Col 0 and 9 are the only ones to be recovered for (int i = 0; i < 10; ++i) { long ts; - if(i==9) + if (i == 9) ts = TimeUnit.MILLISECONDS.toMicros(timeMS - 1000); else ts = TimeUnit.MILLISECONDS.toMicros(timeMS + (i * 1000)); - ColumnFamily cf = ArrayBackedSortedColumns.factory.create(KEYSPACE1, "Standard1"); - cf.addColumn(column("name-" + i, "value", ts)); - Mutation rm = new Mutation(KEYSPACE1, dk.getKey(), cf); - rm.apply(); + new RowUpdateBuilder(cfs.metadata, ts, "name-" + i) + .clustering("cc") + .add("val", Integer.toString(i)) + .build() + .apply(); } - ColumnFamily cf = Util.getColumnFamily(keyspace1, dk, "Standard1"); - Assert.assertEquals(10, cf.getColumnCount()); + // Sanity check row count prior to clear and replay + assertEquals(10, Util.getAll(Util.cmd(cfs).build()).size()); keyspace1.getColumnFamilyStore("Standard1").clearUnsafe(); - CommitLog.instance.resetUnsafe(false); // disassociate segments from live CL + CommitLog.instance.resetUnsafe(false); - cf = Util.getColumnFamily(keyspace1, dk, "Standard1"); - Assert.assertEquals(2, cf.getColumnCount()); + assertEquals(2, Util.getAll(Util.cmd(cfs).build()).size()); } } diff --git a/test/unit/org/apache/cassandra/db/RecoveryManagerTruncateTest.java b/test/unit/org/apache/cassandra/db/RecoveryManagerTruncateTest.java index a004105d27..ee313660af 100644 --- a/test/unit/org/apache/cassandra/db/RecoveryManagerTruncateTest.java +++ b/test/unit/org/apache/cassandra/db/RecoveryManagerTruncateTest.java @@ -18,20 +18,22 @@ */ package org.apache.cassandra.db; -import static org.apache.cassandra.Util.column; -import static org.junit.Assert.*; +import static org.junit.Assert.assertNotNull; +import static org.junit.Assert.assertNull; import java.io.IOException; -import org.junit.BeforeClass; -import org.junit.Test; import org.apache.cassandra.SchemaLoader; import org.apache.cassandra.Util; import org.apache.cassandra.config.KSMetaData; import org.apache.cassandra.db.commitlog.CommitLog; +import org.apache.cassandra.db.partitions.PartitionIterator; import org.apache.cassandra.exceptions.ConfigurationException; import org.apache.cassandra.locator.SimpleStrategy; -import org.apache.cassandra.utils.ByteBufferUtil; +import org.junit.BeforeClass; +import org.junit.Test; + +import static org.junit.Assert.*; /** * Test for the truncate operation. @@ -40,7 +42,6 @@ public class RecoveryManagerTruncateTest { private static final String KEYSPACE1 = "RecoveryManagerTruncateTest"; private static final String CF_STANDARD1 = "Standard1"; - private static final String CF_STANDARD2 = "Standard2"; @BeforeClass public static void defineSchema() throws ConfigurationException @@ -49,151 +50,30 @@ public class RecoveryManagerTruncateTest SchemaLoader.createKeyspace(KEYSPACE1, SimpleStrategy.class, KSMetaData.optsWithRF(1), - SchemaLoader.standardCFMD(KEYSPACE1, CF_STANDARD1), - SchemaLoader.standardCFMD(KEYSPACE1, CF_STANDARD2)); + SchemaLoader.standardCFMD(KEYSPACE1, CF_STANDARD1)); } @Test public void testTruncate() throws IOException { Keyspace keyspace = Keyspace.open(KEYSPACE1); - ColumnFamilyStore cfs = keyspace.getColumnFamilyStore(CF_STANDARD1); - - Mutation rm; - ColumnFamily cf; + ColumnFamilyStore cfs = keyspace.getColumnFamilyStore("Standard1"); // add a single cell - cf = ArrayBackedSortedColumns.factory.create(KEYSPACE1, CF_STANDARD1); - cf.addColumn(column("col1", "val1", 1L)); - rm = new Mutation(KEYSPACE1, ByteBufferUtil.bytes("keymulti"), cf); - rm.applyUnsafe(); - long time = System.currentTimeMillis(); + new RowUpdateBuilder(cfs.metadata, 0, "key1") + .clustering("cc") + .add("val", "val1") + .build() + .applyUnsafe(); // Make sure data was written - assertNotNull(getFromTable(keyspace, CF_STANDARD1, "keymulti", "col1")); + assertTrue(Util.getAll(Util.cmd(cfs).build()).size() > 0); // and now truncate it cfs.truncateBlocking(); - CommitLog.instance.resetUnsafe(false); + assert 0 != CommitLog.instance.resetUnsafe(false); // and validate truncation. - assertNull(getFromTable(keyspace, CF_STANDARD1, "keymulti", "col1")); - assertTrue(SystemKeyspace.getTruncatedAt(cfs.metadata.cfId) > time); - } - - @Test - public void testTruncatePointInTime() throws IOException - { - Keyspace keyspace = Keyspace.open(KEYSPACE1); - ColumnFamilyStore cfs = keyspace.getColumnFamilyStore(CF_STANDARD1); - - Mutation rm; - ColumnFamily cf; - - // add a single cell - cf = ArrayBackedSortedColumns.factory.create(KEYSPACE1, CF_STANDARD1); - cf.addColumn(column("col2", "val1", 1L)); - rm = new Mutation(KEYSPACE1, ByteBufferUtil.bytes("keymulti"), cf); - rm.apply(); - - // Make sure data was written - long time = System.currentTimeMillis(); - assertNotNull(getFromTable(keyspace, CF_STANDARD1, "keymulti", "col2")); - - // and now truncate it - cfs.truncateBlocking(); - - // verify truncation - assertNull(getFromTable(keyspace, CF_STANDARD1, "keymulti", "col2")); - - try - { - // Restore to point in time. - CommitLog.instance.archiver.restorePointInTime = time; - CommitLog.instance.resetUnsafe(false); - } - finally - { - CommitLog.instance.archiver.restorePointInTime = Long.MAX_VALUE; - } - - // Validate pre-truncation data was restored. - assertNotNull(getFromTable(keyspace, CF_STANDARD1, "keymulti", "col2")); - // And that we don't have a truncation record after restore time. - assertFalse(SystemKeyspace.getTruncatedAt(cfs.metadata.cfId) > time); - } - - @Test - public void testTruncatePointInTimeReplayList() throws IOException - { - Keyspace keyspace = Keyspace.open(KEYSPACE1); - ColumnFamilyStore cfs1 = keyspace.getColumnFamilyStore(CF_STANDARD1); - ColumnFamilyStore cfs2 = keyspace.getColumnFamilyStore(CF_STANDARD2); - - Mutation rm; - ColumnFamily cf; - - // add a single cell - cf = ArrayBackedSortedColumns.factory.create(KEYSPACE1, CF_STANDARD1); - cf.addColumn(column("col3", "val1", 1L)); - rm = new Mutation(KEYSPACE1, ByteBufferUtil.bytes("keymulti"), cf); - rm.apply(); - - // add a single cell - cf = ArrayBackedSortedColumns.factory.create(KEYSPACE1, CF_STANDARD2); - cf.addColumn(column("col4", "val1", 1L)); - rm = new Mutation(KEYSPACE1, ByteBufferUtil.bytes("keymulti"), cf); - rm.apply(); - - // Make sure data was written - long time = System.currentTimeMillis(); - assertNotNull(getFromTable(keyspace, CF_STANDARD1, "keymulti", "col3")); - assertNotNull(getFromTable(keyspace, CF_STANDARD2, "keymulti", "col4")); - - // and now truncate it - cfs1.truncateBlocking(); - cfs2.truncateBlocking(); - - // verify truncation - assertNull(getFromTable(keyspace, CF_STANDARD1, "keymulti", "col3")); - assertNull(getFromTable(keyspace, CF_STANDARD2, "keymulti", "col4")); - - try - { - // Restore to point in time. - CommitLog.instance.archiver.restorePointInTime = time; - System.setProperty("cassandra.replayList", KEYSPACE1 + "." + CF_STANDARD1); - CommitLog.instance.resetUnsafe(false); - } - finally - { - CommitLog.instance.archiver.restorePointInTime = Long.MAX_VALUE; - System.clearProperty("cassandra.replayList"); - } - - // Validate pre-truncation data was restored. - assertNotNull(getFromTable(keyspace, CF_STANDARD1, "keymulti", "col3")); - // But only on the replayed table. - assertNull(getFromTable(keyspace, CF_STANDARD2, "keymulti", "col4")); - - // And that we have the correct truncation records. - assertFalse(SystemKeyspace.getTruncatedAt(cfs1.metadata.cfId) > time); - assertTrue(SystemKeyspace.getTruncatedAt(cfs2.metadata.cfId) > time); - } - - private Cell getFromTable(Keyspace keyspace, String cfName, String keyName, String columnName) - { - ColumnFamily cf; - ColumnFamilyStore cfStore = keyspace.getColumnFamilyStore(cfName); - if (cfStore == null) - { - return null; - } - cf = cfStore.getColumnFamily(Util.namesQueryFilter(cfStore, Util.dk(keyName), columnName)); - if (cf == null) - { - return null; - } - return cf.getColumn(Util.cellname(columnName)); + Util.assertEmptyUnfiltered(Util.cmd(cfs).build()); } } diff --git a/test/unit/org/apache/cassandra/db/RemoveCellTest.java b/test/unit/org/apache/cassandra/db/RemoveCellTest.java index 1edb964c31..01fe2551f4 100644 --- a/test/unit/org/apache/cassandra/db/RemoveCellTest.java +++ b/test/unit/org/apache/cassandra/db/RemoveCellTest.java @@ -18,84 +18,21 @@ */ package org.apache.cassandra.db; -import org.junit.BeforeClass; import org.junit.Test; -import static org.junit.Assert.assertFalse; -import static org.junit.Assert.assertNull; +import org.apache.cassandra.cql3.CQLTester; -import org.apache.cassandra.SchemaLoader; -import org.apache.cassandra.Util; -import org.apache.cassandra.config.KSMetaData; -import org.apache.cassandra.db.filter.QueryFilter; -import org.apache.cassandra.exceptions.ConfigurationException; -import org.apache.cassandra.locator.SimpleStrategy; -import org.apache.cassandra.utils.ByteBufferUtil; - -public class RemoveCellTest +public class RemoveCellTest extends CQLTester { - private static final String KEYSPACE1 = "RemoveCellTest"; - private static final String CF_STANDARD1 = "Standard1"; - - @BeforeClass - public static void defineSchema() throws ConfigurationException - { - SchemaLoader.prepareServer(); - SchemaLoader.createKeyspace(KEYSPACE1, - SimpleStrategy.class, - KSMetaData.optsWithRF(1), - SchemaLoader.standardCFMD(KEYSPACE1, CF_STANDARD1)); - } - @Test - public void testRemoveColumn() + public void testDeleteCell() throws Throwable { - Keyspace keyspace = Keyspace.open(KEYSPACE1); - ColumnFamilyStore store = keyspace.getColumnFamilyStore("Standard1"); - Mutation rm; - DecoratedKey dk = Util.dk("key1"); - - // add data - rm = new Mutation(KEYSPACE1, dk.getKey()); - rm.add("Standard1", Util.cellname("Column1"), ByteBufferUtil.bytes("asdf"), 0); - rm.applyUnsafe(); - store.forceBlockingFlush(); - - // remove - rm = new Mutation(KEYSPACE1, dk.getKey()); - rm.delete("Standard1", Util.cellname("Column1"), 1); - rm.applyUnsafe(); - - ColumnFamily retrieved = store.getColumnFamily(Util.namesQueryFilter(store, dk, "Column1")); - assertFalse(retrieved.getColumn(Util.cellname("Column1")).isLive()); - assertNull(Util.cloneAndRemoveDeleted(retrieved, Integer.MAX_VALUE)); - assertNull(Util.cloneAndRemoveDeleted(store.getColumnFamily(QueryFilter.getIdentityFilter(dk, - "Standard1", - System.currentTimeMillis())), - Integer.MAX_VALUE)); + String tableName = createTable("CREATE TABLE %s (a int, b int, c int, PRIMARY KEY (a, b))"); + ColumnFamilyStore cfs = Keyspace.open(KEYSPACE).getColumnFamilyStore(tableName); + execute("INSERT INTO %s (a, b, c) VALUES (?, ?, ?) USING TIMESTAMP ?", 0, 0, 0, 0L); + cfs.forceBlockingFlush(); + execute("DELETE c FROM %s USING TIMESTAMP ? WHERE a = ? AND b = ?", 1L, 0, 0); + assertRows(execute("SELECT * FROM %s WHERE a = ? AND b = ?", 0, 0), row(0, 0, null)); + assertRows(execute("SELECT c FROM %s WHERE a = ? AND b = ?", 0, 0), row(new Object[]{null})); } - - private static BufferDeletedCell dc(String name, int ldt, long timestamp) - { - return new BufferDeletedCell(Util.cellname(name), ldt, timestamp); - } - - @Test - public void deletedColumnShouldAlwaysBeMarkedForDelete() - { - // Check for bug in #4307 - long timestamp = System.currentTimeMillis(); - int localDeletionTime = (int) (timestamp / 1000); - Cell c = dc("dc1", localDeletionTime, timestamp); - assertFalse("DeletedCell was not marked for delete", c.isLive(timestamp)); - - // Simulate a node that is 30 seconds behind - c = dc("dc2", localDeletionTime + 30, timestamp + 30000); - assertFalse("DeletedCell was not marked for delete", c.isLive(timestamp)); - - // Simulate a node that is 30 ahead behind - c = dc("dc3", localDeletionTime - 30, timestamp - 30000); - assertFalse("DeletedCell was not marked for delete", c.isLive(timestamp)); - } - } diff --git a/test/unit/org/apache/cassandra/db/RemoveColumnFamilyTest.java b/test/unit/org/apache/cassandra/db/RemoveColumnFamilyTest.java deleted file mode 100644 index fec8711f93..0000000000 --- a/test/unit/org/apache/cassandra/db/RemoveColumnFamilyTest.java +++ /dev/null @@ -1,73 +0,0 @@ -/* -* Licensed to the Apache Software Foundation (ASF) under one -* or more contributor license agreements. See the NOTICE file -* distributed with this work for additional information -* regarding copyright ownership. The ASF licenses this file -* to you under the Apache License, Version 2.0 (the -* "License"); you may not use this file except in compliance -* with the License. You may obtain a copy of the License at -* -* http://www.apache.org/licenses/LICENSE-2.0 -* -* Unless required by applicable law or agreed to in writing, -* software distributed under the License is distributed on an -* "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -* KIND, either express or implied. See the License for the -* specific language governing permissions and limitations -* under the License. -*/ -package org.apache.cassandra.db; - -import org.junit.BeforeClass; -import org.junit.Test; - -import static org.junit.Assert.assertNull; -import org.apache.cassandra.db.filter.QueryFilter; - -import org.apache.cassandra.SchemaLoader; -import org.apache.cassandra.Util; -import org.apache.cassandra.config.KSMetaData; -import org.apache.cassandra.exceptions.ConfigurationException; -import org.apache.cassandra.locator.SimpleStrategy; -import org.apache.cassandra.utils.ByteBufferUtil; - - -public class RemoveColumnFamilyTest -{ - private static final String KEYSPACE1 = "RemoveColumnFamilyTest"; - private static final String CF_STANDARD1 = "Standard1"; - - @BeforeClass - public static void defineSchema() throws ConfigurationException - { - SchemaLoader.prepareServer(); - SchemaLoader.createKeyspace(KEYSPACE1, - SimpleStrategy.class, - KSMetaData.optsWithRF(1), - SchemaLoader.standardCFMD(KEYSPACE1, CF_STANDARD1)); - } - - @Test - public void testRemoveColumnFamily() - { - Keyspace keyspace = Keyspace.open(KEYSPACE1); - ColumnFamilyStore store = keyspace.getColumnFamilyStore("Standard1"); - Mutation rm; - DecoratedKey dk = Util.dk("key1"); - - // add data - rm = new Mutation(KEYSPACE1, dk.getKey()); - rm.add("Standard1", Util.cellname("Column1"), ByteBufferUtil.bytes("asdf"), 0); - rm.applyUnsafe(); - - // remove - rm = new Mutation(KEYSPACE1, dk.getKey()); - rm.delete("Standard1", 1); - rm.applyUnsafe(); - - ColumnFamily retrieved = store.getColumnFamily(QueryFilter.getIdentityFilter(dk, "Standard1", System.currentTimeMillis())); - assert retrieved.isMarkedForDelete(); - assertNull(retrieved.getColumn(Util.cellname("Column1"))); - assertNull(Util.cloneAndRemoveDeleted(retrieved, Integer.MAX_VALUE)); - } -} diff --git a/test/unit/org/apache/cassandra/db/RemoveColumnFamilyWithFlush1Test.java b/test/unit/org/apache/cassandra/db/RemoveColumnFamilyWithFlush1Test.java deleted file mode 100644 index 72827d06a2..0000000000 --- a/test/unit/org/apache/cassandra/db/RemoveColumnFamilyWithFlush1Test.java +++ /dev/null @@ -1,75 +0,0 @@ -/* -* Licensed to the Apache Software Foundation (ASF) under one -* or more contributor license agreements. See the NOTICE file -* distributed with this work for additional information -* regarding copyright ownership. The ASF licenses this file -* to you under the Apache License, Version 2.0 (the -* "License"); you may not use this file except in compliance -* with the License. You may obtain a copy of the License at -* -* http://www.apache.org/licenses/LICENSE-2.0 -* -* Unless required by applicable law or agreed to in writing, -* software distributed under the License is distributed on an -* "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -* KIND, either express or implied. See the License for the -* specific language governing permissions and limitations -* under the License. -*/ -package org.apache.cassandra.db; - -import org.junit.BeforeClass; -import org.junit.Test; - -import static org.junit.Assert.assertNull; -import org.apache.cassandra.db.filter.QueryFilter; - -import org.apache.cassandra.SchemaLoader; -import org.apache.cassandra.Util; -import org.apache.cassandra.config.KSMetaData; -import org.apache.cassandra.exceptions.ConfigurationException; -import org.apache.cassandra.locator.SimpleStrategy; -import org.apache.cassandra.utils.ByteBufferUtil; - - -public class RemoveColumnFamilyWithFlush1Test -{ - private static final String KEYSPACE1 = "RemoveColumnFamilyWithFlush1Test"; - private static final String CF_STANDARD1 = "Standard1"; - - @BeforeClass - public static void defineSchema() throws ConfigurationException - { - SchemaLoader.prepareServer(); - SchemaLoader.createKeyspace(KEYSPACE1, - SimpleStrategy.class, - KSMetaData.optsWithRF(1), - SchemaLoader.standardCFMD(KEYSPACE1, CF_STANDARD1)); - } - - @Test - public void testRemoveColumnFamilyWithFlush1() - { - Keyspace keyspace = Keyspace.open(KEYSPACE1); - ColumnFamilyStore store = keyspace.getColumnFamilyStore("Standard1"); - Mutation rm; - DecoratedKey dk = Util.dk("key1"); - - // add data - rm = new Mutation(KEYSPACE1, dk.getKey()); - rm.add("Standard1", Util.cellname("Column1"), ByteBufferUtil.bytes("asdf"), 0); - rm.add("Standard1", Util.cellname("Column2"), ByteBufferUtil.bytes("asdf"), 0); - rm.applyUnsafe(); - store.forceBlockingFlush(); - - // remove - rm = new Mutation(KEYSPACE1, dk.getKey()); - rm.delete("Standard1", 1); - rm.applyUnsafe(); - - ColumnFamily retrieved = store.getColumnFamily(QueryFilter.getIdentityFilter(dk, "Standard1", System.currentTimeMillis())); - assert retrieved.isMarkedForDelete(); - assertNull(retrieved.getColumn(Util.cellname("Column1"))); - assertNull(Util.cloneAndRemoveDeleted(retrieved, Integer.MAX_VALUE)); - } -} diff --git a/test/unit/org/apache/cassandra/db/RemoveColumnFamilyWithFlush2Test.java b/test/unit/org/apache/cassandra/db/RemoveColumnFamilyWithFlush2Test.java deleted file mode 100644 index ef7f7f28ed..0000000000 --- a/test/unit/org/apache/cassandra/db/RemoveColumnFamilyWithFlush2Test.java +++ /dev/null @@ -1,73 +0,0 @@ -/* -* Licensed to the Apache Software Foundation (ASF) under one -* or more contributor license agreements. See the NOTICE file -* distributed with this work for additional information -* regarding copyright ownership. The ASF licenses this file -* to you under the Apache License, Version 2.0 (the -* "License"); you may not use this file except in compliance -* with the License. You may obtain a copy of the License at -* -* http://www.apache.org/licenses/LICENSE-2.0 -* -* Unless required by applicable law or agreed to in writing, -* software distributed under the License is distributed on an -* "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -* KIND, either express or implied. See the License for the -* specific language governing permissions and limitations -* under the License. -*/ -package org.apache.cassandra.db; - -import org.junit.BeforeClass; -import org.junit.Test; - -import static org.junit.Assert.assertNull; -import org.apache.cassandra.db.filter.QueryFilter; - -import org.apache.cassandra.SchemaLoader; -import org.apache.cassandra.Util; -import org.apache.cassandra.config.KSMetaData; -import org.apache.cassandra.exceptions.ConfigurationException; -import org.apache.cassandra.locator.SimpleStrategy; -import org.apache.cassandra.utils.ByteBufferUtil; - - -public class RemoveColumnFamilyWithFlush2Test -{ - private static final String KEYSPACE1 = "RemoveColumnFamilyWithFlush2Test"; - private static final String CF_STANDARD1 = "Standard1"; - - @BeforeClass - public static void defineSchema() throws ConfigurationException - { - SchemaLoader.prepareServer(); - SchemaLoader.createKeyspace(KEYSPACE1, - SimpleStrategy.class, - KSMetaData.optsWithRF(1), - SchemaLoader.standardCFMD(KEYSPACE1, CF_STANDARD1)); - } - - @Test - public void testRemoveColumnFamilyWithFlush2() - { - Keyspace keyspace = Keyspace.open(KEYSPACE1); - ColumnFamilyStore store = keyspace.getColumnFamilyStore("Standard1"); - Mutation rm; - DecoratedKey dk = Util.dk("key1"); - - // add data - rm = new Mutation(KEYSPACE1, dk.getKey()); - rm.add("Standard1", Util.cellname("Column1"), ByteBufferUtil.bytes("asdf"), 0); - rm.applyUnsafe(); - // remove - rm = new Mutation(KEYSPACE1, dk.getKey()); - rm.delete("Standard1", 1); - rm.applyUnsafe(); - store.forceBlockingFlush(); - - ColumnFamily retrieved = store.getColumnFamily(QueryFilter.getIdentityFilter(dk, "Standard1", System.currentTimeMillis())); - assert retrieved.isMarkedForDelete(); - assertNull(retrieved.getColumn(Util.cellname("Column1"))); - assertNull(Util.cloneAndRemoveDeleted(retrieved, Integer.MAX_VALUE)); - } -} diff --git a/test/unit/org/apache/cassandra/db/RemoveSubCellTest.java b/test/unit/org/apache/cassandra/db/RemoveSubCellTest.java deleted file mode 100644 index 3fa5c2f375..0000000000 --- a/test/unit/org/apache/cassandra/db/RemoveSubCellTest.java +++ /dev/null @@ -1,119 +0,0 @@ -/* -* Licensed to the Apache Software Foundation (ASF) under one -* or more contributor license agreements. See the NOTICE file -* distributed with this work for additional information -* regarding copyright ownership. The ASF licenses this file -* to you under the Apache License, Version 2.0 (the -* "License"); you may not use this file except in compliance -* with the License. You may obtain a copy of the License at -* -* http://www.apache.org/licenses/LICENSE-2.0 -* -* Unless required by applicable law or agreed to in writing, -* software distributed under the License is distributed on an -* "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -* KIND, either express or implied. See the License for the -* specific language governing permissions and limitations -* under the License. -*/ -package org.apache.cassandra.db; - -import java.nio.ByteBuffer; -import java.util.concurrent.TimeUnit; - -import org.junit.BeforeClass; -import org.junit.Test; - -import static org.junit.Assert.assertFalse; -import static org.junit.Assert.assertNull; -import org.apache.cassandra.config.KSMetaData; -import org.apache.cassandra.db.composites.*; -import org.apache.cassandra.db.filter.QueryFilter; -import org.apache.cassandra.db.marshal.LongType; -import org.apache.cassandra.exceptions.ConfigurationException; -import org.apache.cassandra.locator.SimpleStrategy; - -import static org.apache.cassandra.Util.getBytes; -import org.apache.cassandra.Util; -import org.apache.cassandra.SchemaLoader; -import org.apache.cassandra.utils.ByteBufferUtil; - -import com.google.common.util.concurrent.Uninterruptibles; - - -public class RemoveSubCellTest -{ - private static final String KEYSPACE1 = "RemoveSubCellTest"; - private static final String CF_SUPER1 = "Super1"; - - @BeforeClass - public static void defineSchema() throws ConfigurationException - { - SchemaLoader.prepareServer(); - SchemaLoader.createKeyspace(KEYSPACE1, - SimpleStrategy.class, - KSMetaData.optsWithRF(1), - SchemaLoader.superCFMD(KEYSPACE1, CF_SUPER1, LongType.instance)); - } - - @Test - public void testRemoveSubColumn() - { - Keyspace keyspace = Keyspace.open(KEYSPACE1); - ColumnFamilyStore store = keyspace.getColumnFamilyStore("Super1"); - Mutation rm; - DecoratedKey dk = Util.dk("key1"); - - // add data - rm = new Mutation(KEYSPACE1, dk.getKey()); - Util.addMutation(rm, "Super1", "SC1", 1, "asdf", 0); - rm.applyUnsafe(); - store.forceBlockingFlush(); - - CellName cname = CellNames.compositeDense(ByteBufferUtil.bytes("SC1"), getBytes(1L)); - // remove - rm = new Mutation(KEYSPACE1, dk.getKey()); - rm.delete("Super1", cname, 1); - rm.applyUnsafe(); - - ColumnFamily retrieved = store.getColumnFamily(QueryFilter.getIdentityFilter(dk, "Super1", System.currentTimeMillis())); - assertFalse(retrieved.getColumn(cname).isLive()); - assertNull(Util.cloneAndRemoveDeleted(retrieved, Integer.MAX_VALUE)); - } - - @Test - public void testRemoveSubColumnAndContainer() - { - Keyspace keyspace = Keyspace.open(KEYSPACE1); - ColumnFamilyStore store = keyspace.getColumnFamilyStore("Super1"); - Mutation rm; - DecoratedKey dk = Util.dk("key2"); - - // add data - rm = new Mutation(KEYSPACE1, dk.getKey()); - Util.addMutation(rm, "Super1", "SC1", 1, "asdf", 0); - rm.applyUnsafe(); - store.forceBlockingFlush(); - - // remove the SC - ByteBuffer scName = ByteBufferUtil.bytes("SC1"); - CellName cname = CellNames.compositeDense(scName, getBytes(1L)); - rm = new Mutation(KEYSPACE1, dk.getKey()); - rm.deleteRange("Super1", SuperColumns.startOf(scName), SuperColumns.endOf(scName), 1); - rm.applyUnsafe(); - - // Mark current time and make sure the next insert happens at least - // one second after the previous one (since gc resolution is the second) - QueryFilter filter = QueryFilter.getIdentityFilter(dk, "Super1", System.currentTimeMillis()); - Uninterruptibles.sleepUninterruptibly(1, TimeUnit.SECONDS); - - // remove the column itself - rm = new Mutation(KEYSPACE1, dk.getKey()); - rm.delete("Super1", cname, 2); - rm.applyUnsafe(); - - ColumnFamily retrieved = store.getColumnFamily(filter); - assertFalse(retrieved.getColumn(cname).isLive()); - assertNull(Util.cloneAndRemoveDeleted(retrieved, Integer.MAX_VALUE)); - } -} diff --git a/test/unit/org/apache/cassandra/db/RowCacheCQLTest.java b/test/unit/org/apache/cassandra/db/RowCacheCQLTest.java index 3dc5ce3ae1..cb522d6022 100644 --- a/test/unit/org/apache/cassandra/db/RowCacheCQLTest.java +++ b/test/unit/org/apache/cassandra/db/RowCacheCQLTest.java @@ -30,11 +30,11 @@ public class RowCacheCQLTest extends CQLTester public void test7636() throws Throwable { CacheService.instance.setRowCacheCapacityInMB(1); - createTable("CREATE TABLE %s (p1 bigint, c1 int, PRIMARY KEY (p1, c1)) WITH caching = '{\"keys\":\"NONE\", \"rows_per_partition\":\"ALL\"}'"); - execute("INSERT INTO %s (p1, c1) VALUES (123, 10)"); - assertEmpty(execute("SELECT * FROM %s WHERE p1=123 and c1 > 1000")); - UntypedResultSet res = execute("SELECT * FROM %s WHERE p1=123 and c1 > 0"); + createTable("CREATE TABLE %s (p1 bigint, c1 int, v int, PRIMARY KEY (p1, c1)) WITH caching = '{\"keys\":\"NONE\", \"rows_per_partition\":\"ALL\"}'"); + execute("INSERT INTO %s (p1, c1, v) VALUES (?, ?, ?)", 123L, 10, 12); + assertEmpty(execute("SELECT * FROM %s WHERE p1 = ? and c1 > ?", 123L, 1000)); + UntypedResultSet res = execute("SELECT * FROM %s WHERE p1 = ? and c1 > ?", 123L, 0); assertEquals(1, res.size()); - assertEmpty(execute("SELECT * FROM %s WHERE p1=123 and c1 > 1000")); + assertEmpty(execute("SELECT * FROM %s WHERE p1 = ? and c1 > ?", 123L, 1000)); } } diff --git a/test/unit/org/apache/cassandra/db/RowCacheTest.java b/test/unit/org/apache/cassandra/db/RowCacheTest.java index a4b7514432..2d0303ae3f 100644 --- a/test/unit/org/apache/cassandra/db/RowCacheTest.java +++ b/test/unit/org/apache/cassandra/db/RowCacheTest.java @@ -20,21 +20,28 @@ package org.apache.cassandra.db; import java.net.InetAddress; import java.nio.ByteBuffer; -import java.util.Collection; +import java.util.Arrays; +import java.util.Iterator; import org.junit.AfterClass; import org.junit.BeforeClass; import org.junit.Test; - import org.apache.cassandra.SchemaLoader; import org.apache.cassandra.Util; import org.apache.cassandra.cache.CachingOptions; import org.apache.cassandra.cache.RowCacheKey; +import org.apache.cassandra.config.CFMetaData; import org.apache.cassandra.config.KSMetaData; -import org.apache.cassandra.db.composites.*; +import org.apache.cassandra.config.Schema; +import org.apache.cassandra.db.rows.Unfiltered; +import org.apache.cassandra.db.rows.UnfilteredRowIterator; +import org.apache.cassandra.db.rows.Cell; +import org.apache.cassandra.db.rows.Row; import org.apache.cassandra.db.compaction.CompactionManager; -import org.apache.cassandra.db.filter.QueryFilter; +import org.apache.cassandra.db.filter.ColumnFilter; +import org.apache.cassandra.db.filter.ClusteringIndexNamesFilter; import org.apache.cassandra.db.marshal.IntegerType; +import org.apache.cassandra.db.partitions.CachedPartition; import org.apache.cassandra.exceptions.ConfigurationException; import org.apache.cassandra.dht.ByteOrderedPartitioner.BytesToken; import org.apache.cassandra.locator.TokenMetadata; @@ -42,7 +49,9 @@ import org.apache.cassandra.locator.SimpleStrategy; import org.apache.cassandra.service.CacheService; import org.apache.cassandra.service.StorageService; import org.apache.cassandra.utils.ByteBufferUtil; -import static org.junit.Assert.assertEquals; +import org.apache.cassandra.utils.FBUtilities; + +import static org.junit.Assert.*; public class RowCacheTest { @@ -58,8 +67,7 @@ public class RowCacheTest SimpleStrategy.class, KSMetaData.optsWithRF(1), SchemaLoader.standardCFMD(KEYSPACE_CACHED, CF_CACHED).caching(CachingOptions.ALL), - SchemaLoader.standardCFMD(KEYSPACE_CACHED, CF_CACHEDINT) - .defaultValidator(IntegerType.instance) + SchemaLoader.standardCFMD(KEYSPACE_CACHED, CF_CACHEDINT, 1, IntegerType.instance) .caching(new CachingOptions(new CachingOptions.KeyCache(CachingOptions.KeyCache.Type.ALL), new CachingOptions.RowCache(CachingOptions.RowCache.Type.HEAD, 100)))); } @@ -70,6 +78,54 @@ public class RowCacheTest SchemaLoader.cleanupSavedCaches(); } + @Test + public void testRoundTrip() throws Exception + { + CompactionManager.instance.disableAutoCompaction(); + + Keyspace keyspace = Keyspace.open(KEYSPACE_CACHED); + String cf = "CachedIntCF"; + ColumnFamilyStore cachedStore = keyspace.getColumnFamilyStore(cf); + long startRowCacheHits = cachedStore.metric.rowCacheHit.getCount(); + long startRowCacheOutOfRange = cachedStore.metric.rowCacheHitOutOfRange.getCount(); + // empty the row cache + CacheService.instance.invalidateRowCache(); + + // set global row cache size to 1 MB + CacheService.instance.setRowCacheCapacityInMB(1); + + ByteBuffer key = ByteBufferUtil.bytes("rowcachekey"); + DecoratedKey dk = cachedStore.partitioner.decorateKey(key); + RowCacheKey rck = new RowCacheKey(cachedStore.metadata.cfId, dk); + + RowUpdateBuilder rub = new RowUpdateBuilder(cachedStore.metadata, System.currentTimeMillis(), key); + rub.clustering(String.valueOf(0)); + rub.add("val", ByteBufferUtil.bytes("val" + 0)); + rub.build().applyUnsafe(); + + // populate row cache, we should not get a row cache hit; + Util.getAll(Util.cmd(cachedStore, dk).withLimit(1).build()); + assertEquals(startRowCacheHits, cachedStore.metric.rowCacheHit.getCount()); + + // do another query, limit is 20, which is < 100 that we cache, we should get a hit and it should be in range + Util.getAll(Util.cmd(cachedStore, dk).withLimit(1).build()); + assertEquals(++startRowCacheHits, cachedStore.metric.rowCacheHit.getCount()); + assertEquals(startRowCacheOutOfRange, cachedStore.metric.rowCacheHitOutOfRange.getCount()); + + CachedPartition cachedCf = (CachedPartition)CacheService.instance.rowCache.get(rck); + assertEquals(1, cachedCf.rowCount()); + for (Unfiltered unfiltered : Util.once(cachedCf.unfilteredIterator(ColumnFilter.selection(cachedCf.columns()), Slices.ALL, false))) + { + Row r = (Row) unfiltered; + + for (Cell c : r) + { + assertEquals(c.value(), ByteBufferUtil.bytes("val" + 0)); + } + } + cachedStore.truncateBlocking(); + } + @Test public void testRowCache() throws Exception { @@ -92,19 +148,25 @@ public class RowCacheTest { DecoratedKey key = Util.dk("key" + i); - cachedStore.getColumnFamily(key, Composites.EMPTY, Composites.EMPTY, false, 1, System.currentTimeMillis()); + Util.getAll(Util.cmd(cachedStore, key).build()); assert CacheService.instance.rowCache.size() == i + 1; - assert cachedStore.containsCachedRow(key); // current key should be stored in the cache + assert cachedStore.containsCachedParition(key); // current key should be stored in the cache // checking if cell is read correctly after cache - ColumnFamily cf = cachedStore.getColumnFamily(key, Composites.EMPTY, Composites.EMPTY, false, 1, System.currentTimeMillis()); - Collection cells = cf.getSortedColumns(); + CachedPartition cp = cachedStore.getRawCachedPartition(key); + try (UnfilteredRowIterator ai = cp.unfilteredIterator(ColumnFilter.selection(cp.columns()), Slices.ALL, false)) + { + assert ai.hasNext(); + Row r = (Row)ai.next(); + assertFalse(ai.hasNext()); - Cell cell = cells.iterator().next(); + Iterator ci = r.iterator(); + assert(ci.hasNext()); + Cell cell = ci.next(); - assert cells.size() == 1; - assert cell.name().toByteBuffer().equals(ByteBufferUtil.bytes("col" + i)); - assert cell.value().equals(ByteBufferUtil.bytes("val" + i)); + assert cell.column().name.bytes.equals(ByteBufferUtil.bytes("val")); + assert cell.value().equals(ByteBufferUtil.bytes("val" + i)); + } } // insert 10 more keys @@ -114,25 +176,31 @@ public class RowCacheTest { DecoratedKey key = Util.dk("key" + i); - cachedStore.getColumnFamily(key, Composites.EMPTY, Composites.EMPTY, false, 1, System.currentTimeMillis()); - assert cachedStore.containsCachedRow(key); // cache should be populated with the latest rows read (old ones should be popped) + Util.getAll(Util.cmd(cachedStore, key).build()); + assert cachedStore.containsCachedParition(key); // cache should be populated with the latest rows read (old ones should be popped) // checking if cell is read correctly after cache - ColumnFamily cf = cachedStore.getColumnFamily(key, Composites.EMPTY, Composites.EMPTY, false, 1, System.currentTimeMillis()); - Collection cells = cf.getSortedColumns(); + CachedPartition cp = cachedStore.getRawCachedPartition(key); + try (UnfilteredRowIterator ai = cp.unfilteredIterator(ColumnFilter.selection(cp.columns()), Slices.ALL, false)) + { + assert ai.hasNext(); + Row r = (Row)ai.next(); + assertFalse(ai.hasNext()); - Cell cell = cells.iterator().next(); + Iterator ci = r.iterator(); + assert(ci.hasNext()); + Cell cell = ci.next(); - assert cells.size() == 1; - assert cell.name().toByteBuffer().equals(ByteBufferUtil.bytes("col" + i)); - assert cell.value().equals(ByteBufferUtil.bytes("val" + i)); + assert cell.column().name.bytes.equals(ByteBufferUtil.bytes("val")); + assert cell.value().equals(ByteBufferUtil.bytes("val" + i)); + } } // clear 100 rows from the cache int keysLeft = 109; for (int i = 109; i >= 10; i--) { - cachedStore.invalidateCachedRow(Util.dk("key" + i)); + cachedStore.invalidateCachedPartition(Util.dk("key" + i)); assert CacheService.instance.rowCache.size() == keysLeft; keysLeft--; } @@ -177,6 +245,7 @@ public class RowCacheTest rowCacheLoad(100, 50, 0); CacheService.instance.setRowCacheCapacityInMB(0); } + @Test public void testRowCacheRange() { @@ -196,39 +265,33 @@ public class RowCacheTest ByteBuffer key = ByteBufferUtil.bytes("rowcachekey"); DecoratedKey dk = cachedStore.partitioner.decorateKey(key); RowCacheKey rck = new RowCacheKey(cachedStore.metadata.cfId, dk); - Mutation mutation = new Mutation(KEYSPACE_CACHED, key); + String values[] = new String[200]; for (int i = 0; i < 200; i++) - mutation.add(cf, Util.cellname(i), ByteBufferUtil.bytes("val" + i), System.currentTimeMillis()); - mutation.applyUnsafe(); + { + RowUpdateBuilder rub = new RowUpdateBuilder(cachedStore.metadata, System.currentTimeMillis(), key); + rub.clustering(String.valueOf(i)); + values[i] = "val" + i; + rub.add("val", ByteBufferUtil.bytes(values[i])); + rub.build().applyUnsafe(); + } + Arrays.sort(values); // populate row cache, we should not get a row cache hit; - cachedStore.getColumnFamily(QueryFilter.getSliceFilter(dk, cf, - Composites.EMPTY, - Composites.EMPTY, - false, 10, System.currentTimeMillis())); + Util.getAll(Util.cmd(cachedStore, dk).withLimit(10).build()); assertEquals(startRowCacheHits, cachedStore.metric.rowCacheHit.getCount()); // do another query, limit is 20, which is < 100 that we cache, we should get a hit and it should be in range - cachedStore.getColumnFamily(QueryFilter.getSliceFilter(dk, cf, - Composites.EMPTY, - Composites.EMPTY, - false, 20, System.currentTimeMillis())); + Util.getAll(Util.cmd(cachedStore, dk).withLimit(10).build()); assertEquals(++startRowCacheHits, cachedStore.metric.rowCacheHit.getCount()); assertEquals(startRowCacheOutOfRange, cachedStore.metric.rowCacheHitOutOfRange.getCount()); // get a slice from 95 to 105, 95->99 are in cache, we should not get a hit and then row cache is out of range - cachedStore.getColumnFamily(QueryFilter.getSliceFilter(dk, cf, - CellNames.simpleDense(ByteBufferUtil.bytes(95)), - CellNames.simpleDense(ByteBufferUtil.bytes(105)), - false, 10, System.currentTimeMillis())); + Util.getAll(Util.cmd(cachedStore, dk).fromIncl(String.valueOf(210)).toExcl(String.valueOf(215)).build()); assertEquals(startRowCacheHits, cachedStore.metric.rowCacheHit.getCount()); assertEquals(++startRowCacheOutOfRange, cachedStore.metric.rowCacheHitOutOfRange.getCount()); // get a slice with limit > 100, we should get a hit out of range. - cachedStore.getColumnFamily(QueryFilter.getSliceFilter(dk, cf, - Composites.EMPTY, - Composites.EMPTY, - false, 101, System.currentTimeMillis())); + Util.getAll(Util.cmd(cachedStore, dk).withLimit(101).build()); assertEquals(startRowCacheHits, cachedStore.metric.rowCacheHit.getCount()); assertEquals(++startRowCacheOutOfRange, cachedStore.metric.rowCacheHitOutOfRange.getCount()); @@ -236,19 +299,28 @@ public class RowCacheTest CacheService.instance.invalidateRowCache(); // try to populate row cache with a limit > rows to cache, we should still populate row cache; - cachedStore.getColumnFamily(QueryFilter.getSliceFilter(dk, cf, - Composites.EMPTY, - Composites.EMPTY, - false, 105, System.currentTimeMillis())); + Util.getAll(Util.cmd(cachedStore, dk).withLimit(105).build()); assertEquals(startRowCacheHits, cachedStore.metric.rowCacheHit.getCount()); + // validate the stuff in cache; - ColumnFamily cachedCf = (ColumnFamily)CacheService.instance.rowCache.get(rck); - assertEquals(cachedCf.getColumnCount(), 100); + CachedPartition cachedCf = (CachedPartition)CacheService.instance.rowCache.get(rck); + assertEquals(cachedCf.rowCount(), 100); int i = 0; - for(Cell c : cachedCf) + + for (Unfiltered unfiltered : Util.once(cachedCf.unfilteredIterator(ColumnFilter.selection(cachedCf.columns()), Slices.ALL, false))) { - assertEquals(c.name(), Util.cellname(i++)); + Row r = (Row) unfiltered; + + assertEquals(r.clustering().get(0), ByteBufferUtil.bytes(values[i].substring(3))); + + for (Cell c : r) + { + assertEquals(c.value(), ByteBufferUtil.bytes(values[i])); + } + i++; } + + cachedStore.truncateBlocking(); } public void rowCacheLoad(int totalKeys, int keysToSave, int offset) throws Exception @@ -263,7 +335,7 @@ public class RowCacheTest // insert data and fill the cache SchemaLoader.insertData(KEYSPACE_CACHED, CF_CACHED, offset, totalKeys); - SchemaLoader.readData(KEYSPACE_CACHED, CF_CACHED, offset, totalKeys); + readData(KEYSPACE_CACHED, CF_CACHED, offset, totalKeys); assertEquals(totalKeys, CacheService.instance.rowCache.size()); // force the cache to disk @@ -274,4 +346,17 @@ public class RowCacheTest assertEquals(0, CacheService.instance.rowCache.size()); assertEquals(keysToSave == Integer.MAX_VALUE ? totalKeys : keysToSave, CacheService.instance.rowCache.loadSaved(store)); } + + private static void readData(String keyspace, String columnFamily, int offset, int numberOfRows) + { + ColumnFamilyStore store = Keyspace.open(keyspace).getColumnFamilyStore(columnFamily); + CFMetaData cfm = Schema.instance.getCFMetaData(keyspace, columnFamily); + + for (int i = offset; i < offset + numberOfRows; i++) + { + DecoratedKey key = Util.dk("key" + i); + Clustering cl = new SimpleClustering(ByteBufferUtil.bytes("col" + i)); + Util.getAll(Util.cmd(store, key).build()); + } + } } diff --git a/test/unit/org/apache/cassandra/db/RowIndexEntryTest.java b/test/unit/org/apache/cassandra/db/RowIndexEntryTest.java index e880d95c3a..99aa5e07d3 100644 --- a/test/unit/org/apache/cassandra/db/RowIndexEntryTest.java +++ b/test/unit/org/apache/cassandra/db/RowIndexEntryTest.java @@ -17,63 +17,58 @@ */ package org.apache.cassandra.db; -import java.io.IOException; -import java.util.Collections; +import java.io.File; -import junit.framework.Assert; -import org.apache.cassandra.SchemaLoader; +import org.apache.cassandra.Util; import org.apache.cassandra.config.DatabaseDescriptor; -import org.apache.cassandra.config.KSMetaData; -import org.apache.cassandra.config.Schema; -import org.apache.cassandra.db.composites.CellNames; -import org.apache.cassandra.db.composites.SimpleDenseCellNameType; -import org.apache.cassandra.db.marshal.UTF8Type; +import org.apache.cassandra.cql3.CQLTester; +import org.apache.cassandra.db.rows.UnfilteredRowIterator; +import org.apache.cassandra.db.rows.RowStats; +import org.apache.cassandra.db.partitions.*; import org.apache.cassandra.io.sstable.IndexHelper; +import org.apache.cassandra.io.sstable.format.big.BigFormat; import org.apache.cassandra.io.util.DataOutputBuffer; -import org.apache.cassandra.locator.SimpleStrategy; -import org.apache.cassandra.utils.ByteBufferUtil; -import org.apache.cassandra.utils.FBUtilities; +import org.apache.cassandra.io.util.SequentialWriter; import org.junit.Test; -public class RowIndexEntryTest extends SchemaLoader +import static junit.framework.Assert.assertEquals; +import static junit.framework.Assert.assertTrue; + +public class RowIndexEntryTest extends CQLTester { @Test - public void testSerializedSize() throws IOException + public void testSerializedSize() throws Throwable { - final RowIndexEntry simple = new RowIndexEntry<>(123); + String tableName = createTable("CREATE TABLE %s (a int, b text, c int, PRIMARY KEY(a, b))"); + ColumnFamilyStore cfs = Keyspace.open(KEYSPACE).getColumnFamilyStore(tableName); + + final RowIndexEntry simple = new RowIndexEntry(123); DataOutputBuffer buffer = new DataOutputBuffer(); - RowIndexEntry.Serializer serializer = new RowIndexEntry.Serializer(new IndexHelper.IndexInfo.Serializer(new SimpleDenseCellNameType(UTF8Type.instance))); + SerializationHeader header = new SerializationHeader(cfs.metadata, cfs.metadata.partitionColumns(), RowStats.NO_STATS); + RowIndexEntry.Serializer serializer = new RowIndexEntry.Serializer(cfs.metadata, BigFormat.latestVersion, header); serializer.serialize(simple, buffer); - Assert.assertEquals(buffer.getLength(), serializer.serializedSize(simple)); + assertEquals(buffer.getLength(), serializer.serializedSize(simple)); + + // write enough rows to ensure we get a few column index entries + for (int i = 0; i <= DatabaseDescriptor.getColumnIndexSize() / 4; i++) + execute("INSERT INTO %s (a, b, c) VALUES (?, ?, ?)", 0, "" + i, i); buffer = new DataOutputBuffer(); - Schema.instance.setKeyspaceDefinition(KSMetaData.newKeyspace("Keyspace1", - SimpleStrategy.class, - Collections.emptyMap(), - false, - Collections.singleton(standardCFMD("Keyspace1", "Standard1")))); - ColumnFamily cf = ArrayBackedSortedColumns.factory.create("Keyspace1", "Standard1"); - ColumnIndex columnIndex = new ColumnIndex.Builder(cf, ByteBufferUtil.bytes("a"), new DataOutputBuffer()) - {{ - int idx = 0, size = 0; - Cell column; - do - { - column = new BufferCell(CellNames.simpleDense(ByteBufferUtil.bytes("c" + idx++)), ByteBufferUtil.bytes("v"), FBUtilities.timestampMicros()); - size += column.serializedSize(new SimpleDenseCellNameType(UTF8Type.instance), TypeSizes.NATIVE); - - add(column); - } - while (size < DatabaseDescriptor.getColumnIndexSize() * 3); - - }}.build(); + ArrayBackedPartition partition = Util.getOnlyPartitionUnfiltered(Util.cmd(cfs).build()); + File tempFile = File.createTempFile("row_index_entry_test", null); + tempFile.deleteOnExit(); + SequentialWriter writer = SequentialWriter.open(tempFile); + ColumnIndex columnIndex = ColumnIndex.writeAndBuildIndex(partition.unfilteredIterator(), writer, header, BigFormat.latestVersion); RowIndexEntry withIndex = RowIndexEntry.create(0xdeadbeef, DeletionTime.LIVE, columnIndex); + // sanity check + assertTrue(columnIndex.columnsIndex.size() >= 3); + serializer.serialize(withIndex, buffer); - Assert.assertEquals(buffer.getLength(), serializer.serializedSize(withIndex)); + assertEquals(buffer.getLength(), serializer.serializedSize(withIndex)); } } diff --git a/test/unit/org/apache/cassandra/db/RowIterationTest.java b/test/unit/org/apache/cassandra/db/RowIterationTest.java index ee7bf1a3cc..3b0293cdae 100644 --- a/test/unit/org/apache/cassandra/db/RowIterationTest.java +++ b/test/unit/org/apache/cassandra/db/RowIterationTest.java @@ -18,105 +18,65 @@ */ package org.apache.cassandra.db; -import java.net.InetAddress; -import java.nio.ByteBuffer; -import java.util.Set; -import java.util.HashSet; - -import org.apache.cassandra.Util; - -import org.junit.BeforeClass; import org.junit.Test; -import org.apache.cassandra.SchemaLoader; -import org.apache.cassandra.config.KSMetaData; -import org.apache.cassandra.db.composites.*; -import org.apache.cassandra.db.marshal.LongType; -import org.apache.cassandra.exceptions.ConfigurationException; -import org.apache.cassandra.locator.SimpleStrategy; -import org.apache.cassandra.utils.FBUtilities; +import org.apache.cassandra.cql3.CQLTester; +import org.apache.cassandra.db.rows.UnfilteredRowIterator; +import org.apache.cassandra.db.partitions.UnfilteredPartitionIterator; +import org.apache.cassandra.Util; + import static org.junit.Assert.assertEquals; -import org.apache.cassandra.utils.ByteBufferUtil; +import static org.junit.Assert.assertTrue; +import static org.junit.Assert.assertFalse; -public class RowIterationTest +public class RowIterationTest extends CQLTester { - public static final String KEYSPACE1 = "RowIterationTest"; - public static final InetAddress LOCAL = FBUtilities.getBroadcastAddress(); - - @BeforeClass - public static void defineSchema() throws ConfigurationException + @Test + public void testRowIteration() throws Throwable { - SchemaLoader.prepareServer(); - SchemaLoader.createKeyspace(KEYSPACE1, - SimpleStrategy.class, - KSMetaData.optsWithRF(1), - SchemaLoader.standardCFMD(KEYSPACE1, "Standard3"), - SchemaLoader.superCFMD(KEYSPACE1, "Super3", LongType.instance)); + String tableName = createTable("CREATE TABLE %s (a int, b int, c int, d int, PRIMARY KEY (a, b, c))"); + ColumnFamilyStore cfs = Keyspace.open(KEYSPACE).getColumnFamilyStore(tableName); + for (int i = 0; i < 10; i++) + execute("INSERT INTO %s (a, b, c, d) VALUES (?, ?, ?, ?) USING TIMESTAMP ?", i, 0, i, i, (long)i); + cfs.forceBlockingFlush(); + assertEquals(10, execute("SELECT * FROM %s").size()); } @Test - public void testRowIteration() + public void testRowIterationDeletionTime() throws Throwable { - Keyspace keyspace = Keyspace.open(KEYSPACE1); - ColumnFamilyStore store = keyspace.getColumnFamilyStore("Super3"); + String tableName = createTable("CREATE TABLE %s (a int PRIMARY KEY, b int)"); + ColumnFamilyStore cfs = Keyspace.open(KEYSPACE).getColumnFamilyStore(tableName); - final int ROWS_PER_SSTABLE = 10; - Set inserted = new HashSet(); - for (int i = 0; i < ROWS_PER_SSTABLE; i++) { - DecoratedKey key = Util.dk(String.valueOf(i)); - Mutation rm = new Mutation(KEYSPACE1, key.getKey()); - rm.add("Super3", CellNames.compositeDense(ByteBufferUtil.bytes("sc"), ByteBufferUtil.bytes(String.valueOf(i))), ByteBuffer.wrap(new byte[ROWS_PER_SSTABLE * 10 - i * 2]), i); - rm.applyUnsafe(); - inserted.add(key); - } - store.forceBlockingFlush(); - assertEquals(inserted.toString(), inserted.size(), Util.getRangeSlice(store).size()); - } + execute("INSERT INTO %s (a, b) VALUES (?, ?) USING TIMESTAMP ?", 0, 0, 0L); + execute("DELETE FROM %s USING TIMESTAMP ? WHERE a = ?", 0L, 0); - @Test - public void testRowIterationDeletionTime() - { - Keyspace keyspace = Keyspace.open(KEYSPACE1); - String CF_NAME = "Standard3"; - ColumnFamilyStore store = keyspace.getColumnFamilyStore(CF_NAME); - DecoratedKey key = Util.dk("key"); - - // Delete row in first sstable - Mutation rm = new Mutation(KEYSPACE1, key.getKey()); - rm.delete(CF_NAME, 0); - rm.add(CF_NAME, Util.cellname("c"), ByteBufferUtil.bytes("values"), 0L); - rm.applyUnsafe(); - store.forceBlockingFlush(); + cfs.forceBlockingFlush(); // Delete row in second sstable with higher timestamp - rm = new Mutation(KEYSPACE1, key.getKey()); - rm.delete(CF_NAME, 1); - rm.add(CF_NAME, Util.cellname("c"), ByteBufferUtil.bytes("values"), 1L); - DeletionInfo delInfo2 = rm.getColumnFamilies().iterator().next().deletionInfo(); - assert delInfo2.getTopLevelDeletion().markedForDeleteAt == 1L; - rm.applyUnsafe(); - store.forceBlockingFlush(); + execute("INSERT INTO %s (a, b) VALUES (?, ?) USING TIMESTAMP ?", 0, 0, 1L); + execute("DELETE FROM %s USING TIMESTAMP ? WHERE a = ?", 1L, 0); - ColumnFamily cf = Util.getRangeSlice(store).get(0).cf; - assert cf.deletionInfo().equals(delInfo2); + int localDeletionTime = Util.getOnlyPartitionUnfiltered(Util.cmd(cfs).build()).partitionLevelDeletion().localDeletionTime(); + + cfs.forceBlockingFlush(); + + DeletionTime dt = Util.getOnlyPartitionUnfiltered(Util.cmd(cfs).build()).partitionLevelDeletion(); + assertEquals(1L, dt.markedForDeleteAt()); + assertEquals(localDeletionTime, dt.localDeletionTime()); } @Test - public void testRowIterationDeletion() + public void testRowIterationDeletion() throws Throwable { - Keyspace keyspace = Keyspace.open(KEYSPACE1); - String CF_NAME = "Standard3"; - ColumnFamilyStore store = keyspace.getColumnFamilyStore(CF_NAME); - DecoratedKey key = Util.dk("key"); + String tableName = createTable("CREATE TABLE %s (a int PRIMARY KEY, b int)"); + ColumnFamilyStore cfs = Keyspace.open(KEYSPACE).getColumnFamilyStore(tableName); // Delete a row in first sstable - Mutation rm = new Mutation(KEYSPACE1, key.getKey()); - rm.delete(CF_NAME, 0); - rm.applyUnsafe(); - store.forceBlockingFlush(); + execute("DELETE FROM %s USING TIMESTAMP ? WHERE a = ?", 0L, 0); + cfs.forceBlockingFlush(); - ColumnFamily cf = Util.getRangeSlice(store).get(0).cf; - assert cf != null; + assertFalse(Util.getOnlyPartitionUnfiltered(Util.cmd(cfs).build()).isEmpty()); } } diff --git a/test/unit/org/apache/cassandra/db/RowTest.java b/test/unit/org/apache/cassandra/db/RowTest.java index 910f9e1166..73a4ed72c3 100644 --- a/test/unit/org/apache/cassandra/db/RowTest.java +++ b/test/unit/org/apache/cassandra/db/RowTest.java @@ -18,22 +18,29 @@ */ package org.apache.cassandra.db; -import java.util.Arrays; -import java.util.concurrent.TimeUnit; +import java.io.IOException; -import com.google.common.util.concurrent.Uninterruptibles; +import com.google.common.collect.ImmutableList; +import org.junit.Before; import org.junit.BeforeClass; import org.junit.Test; import org.apache.cassandra.SchemaLoader; +import org.apache.cassandra.Util; +import org.apache.cassandra.config.CFMetaData; +import org.apache.cassandra.config.ColumnDefinition; import org.apache.cassandra.config.KSMetaData; -import org.apache.cassandra.db.composites.CellNames; +import org.apache.cassandra.cql3.ColumnIdentifier; +import org.apache.cassandra.db.rows.*; +import org.apache.cassandra.db.marshal.AbstractType; +import org.apache.cassandra.db.marshal.AsciiType; +import org.apache.cassandra.db.marshal.BytesType; +import org.apache.cassandra.db.partitions.PartitionUpdate; import org.apache.cassandra.exceptions.ConfigurationException; import org.apache.cassandra.locator.SimpleStrategy; import org.apache.cassandra.utils.ByteBufferUtil; +import org.apache.cassandra.utils.FBUtilities; -import static org.apache.cassandra.Util.column; -import static org.apache.cassandra.Util.tombstone; import static org.junit.Assert.assertEquals; import static org.junit.Assert.assertFalse; import static org.junit.Assert.assertTrue; @@ -43,88 +50,158 @@ public class RowTest private static final String KEYSPACE1 = "RowTest"; private static final String CF_STANDARD1 = "Standard1"; + private int nowInSeconds; + private DecoratedKey dk; + private ColumnFamilyStore cfs; + private CFMetaData cfm; + @BeforeClass public static void defineSchema() throws ConfigurationException { + CFMetaData cfMetadata = CFMetaData.Builder.create(KEYSPACE1, CF_STANDARD1) + .addPartitionKey("key", BytesType.instance) + .addClusteringColumn("col1", AsciiType.instance) + .addRegularColumn("a", AsciiType.instance) + .addRegularColumn("b", AsciiType.instance) + .build(); SchemaLoader.prepareServer(); SchemaLoader.createKeyspace(KEYSPACE1, SimpleStrategy.class, KSMetaData.optsWithRF(1), - SchemaLoader.standardCFMD(KEYSPACE1, CF_STANDARD1)); + cfMetadata); + } + + @Before + public void setup() + { + nowInSeconds = FBUtilities.nowInSeconds(); + dk = Util.dk("key0"); + cfs = Keyspace.open(KEYSPACE1).getColumnFamilyStore(CF_STANDARD1); + cfm = cfs.metadata; } @Test - public void testDiffColumnFamily() + public void testMergeRangeTombstones() throws InterruptedException { - ColumnFamily cf1 = ArrayBackedSortedColumns.factory.create(KEYSPACE1, "Standard1"); - cf1.addColumn(column("one", "onev", 0)); + PartitionUpdate update1 = new PartitionUpdate(cfm, dk, cfm.partitionColumns(), 1); + writeRangeTombstone(update1, "1", "11", 123, 123); + writeRangeTombstone(update1, "2", "22", 123, 123); + writeRangeTombstone(update1, "3", "31", 123, 123); + writeRangeTombstone(update1, "4", "41", 123, 123); - ColumnFamily cf2 = ArrayBackedSortedColumns.factory.create(KEYSPACE1, "Standard1"); - DeletionInfo delInfo = new DeletionInfo(0, 0); - cf2.delete(delInfo); + PartitionUpdate update2 = new PartitionUpdate(cfm, dk, cfm.partitionColumns(), 1); + writeRangeTombstone(update2, "1", "11", 123, 123); + writeRangeTombstone(update2, "111", "112", 1230, 123); + writeRangeTombstone(update2, "2", "24", 123, 123); + writeRangeTombstone(update2, "3", "31", 1230, 123); + writeRangeTombstone(update2, "4", "41", 123, 1230); + writeRangeTombstone(update2, "5", "51", 123, 1230); - ColumnFamily cfDiff = cf1.diff(cf2); - assertFalse(cfDiff.hasColumns()); - assertEquals(cfDiff.deletionInfo(), delInfo); + try (UnfilteredRowIterator merged = UnfilteredRowIterators.merge(ImmutableList.of(update1.unfilteredIterator(), update2.unfilteredIterator()), nowInSeconds)) + { + Object[][] expected = new Object[][]{ { "1", "11", 123l, 123 }, + { "111", "112", 1230l, 123 }, + { "2", "24", 123l, 123 }, + { "3", "31", 1230l, 123 }, + { "4", "41", 123l, 1230 }, + { "5", "51", 123l, 1230 } }; + int i = 0; + while (merged.hasNext()) + { + RangeTombstoneBoundMarker openMarker = (RangeTombstoneBoundMarker)merged.next(); + Slice.Bound openBound = openMarker.clustering(); + DeletionTime openDeletion = new SimpleDeletionTime(openMarker.deletionTime().markedForDeleteAt(), + openMarker.deletionTime().localDeletionTime()); - RangeTombstone tombstone1 = tombstone("1", "11", (long) 123, 123); - RangeTombstone tombstone1_2 = tombstone("111", "112", (long) 1230, 123); - RangeTombstone tombstone2_1 = tombstone("2", "22", (long) 123, 123); - RangeTombstone tombstone2_2 = tombstone("2", "24", (long) 123, 123); - RangeTombstone tombstone3_1 = tombstone("3", "31", (long) 123, 123); - RangeTombstone tombstone3_2 = tombstone("3", "31", (long) 1230, 123); - RangeTombstone tombstone4_1 = tombstone("4", "41", (long) 123, 123); - RangeTombstone tombstone4_2 = tombstone("4", "41", (long) 123, 1230); - RangeTombstone tombstone5_2 = tombstone("5", "51", (long) 123, 1230); - cf1.delete(tombstone1); - cf1.delete(tombstone2_1); - cf1.delete(tombstone3_1); - cf1.delete(tombstone4_1); + RangeTombstoneBoundMarker closeMarker = (RangeTombstoneBoundMarker)merged.next(); + Slice.Bound closeBound = closeMarker.clustering(); + DeletionTime closeDeletion = new SimpleDeletionTime(closeMarker.deletionTime().markedForDeleteAt(), + closeMarker.deletionTime().localDeletionTime()); - cf2.delete(tombstone1); - cf2.delete(tombstone1_2); - cf2.delete(tombstone2_2); - cf2.delete(tombstone3_2); - cf2.delete(tombstone4_2); - cf2.delete(tombstone5_2); - - cfDiff = cf1.diff(cf2); - assertEquals(0, cfDiff.getColumnCount()); - - // only tmbstones which differ in superset or have more recent timestamp to be in diff - delInfo.add(tombstone1_2, cf1.getComparator()); - delInfo.add(tombstone2_2, cf1.getComparator()); - delInfo.add(tombstone3_2, cf1.getComparator()); - delInfo.add(tombstone5_2, cf1.getComparator()); - - assertEquals(delInfo, cfDiff.deletionInfo()); + assertEquals(openDeletion, closeDeletion); + assertRangeTombstoneMarkers(openBound, closeBound, openDeletion, expected[i++]); + } + } } @Test public void testResolve() { - ColumnFamily cf1 = ArrayBackedSortedColumns.factory.create(KEYSPACE1, "Standard1"); - cf1.addColumn(column("one", "A", 0)); + ColumnDefinition defA = cfm.getColumnDefinition(new ColumnIdentifier("a", true)); + ColumnDefinition defB = cfm.getColumnDefinition(new ColumnIdentifier("b", true)); - ColumnFamily cf2 = ArrayBackedSortedColumns.factory.create(KEYSPACE1, "Standard1"); - cf2.addColumn(column("one", "B", 1)); - cf2.addColumn(column("two", "C", 1)); + PartitionUpdate update = new PartitionUpdate(cfm, dk, cfm.partitionColumns(), 1); + Rows.writeClustering(update.metadata().comparator.make("c1"), update.writer()); + writeSimpleCellValue(update.writer(), cfm, defA, "a1", 0, nowInSeconds); + writeSimpleCellValue(update.writer(), cfm, defA, "a2", 1, nowInSeconds); + writeSimpleCellValue(update.writer(), cfm, defB, "b1", 1, nowInSeconds); + update.writer().endOfRow(); - cf1.addAll(cf2); - assert Arrays.equals(cf1.getColumn(CellNames.simpleDense(ByteBufferUtil.bytes("one"))).value().array(), "B".getBytes()); - assert Arrays.equals(cf1.getColumn(CellNames.simpleDense(ByteBufferUtil.bytes("two"))).value().array(), "C".getBytes()); + Unfiltered unfiltered = update.unfilteredIterator().next(); + assertTrue(unfiltered.kind() == Unfiltered.Kind.ROW); + Row row = (Row) unfiltered; + assertEquals("a2", defA.cellValueType().getString(row.getCell(defA).value())); + assertEquals("b1", defB.cellValueType().getString(row.getCell(defB).value())); + assertEquals(2, row.columns().columnCount()); } @Test - public void testExpiringColumnExpiration() + public void testExpiringColumnExpiration() throws IOException { - Cell c = new BufferExpiringCell(CellNames.simpleDense(ByteBufferUtil.bytes("one")), ByteBufferUtil.bytes("A"), 0, 1); - assertTrue(c.isLive()); + int ttl = 1; + ColumnDefinition def = cfm.getColumnDefinition(new ColumnIdentifier("a", true)); + PartitionUpdate update = new PartitionUpdate(cfm, dk, cfm.partitionColumns(), 1); + Rows.writeClustering(update.metadata().comparator.make("c1"), update.writer()); + update.writer().writeCell(def, false, ((AbstractType) def.cellValueType()).decompose("a1"), + SimpleLivenessInfo.forUpdate(0, ttl, nowInSeconds, cfm), + null); + update.writer().endOfRow(); + new Mutation(update).applyUnsafe(); - // Because we keep the local deletion time with a precision of a - // second, we could have to wait 2 seconds in worst case scenario. - Uninterruptibles.sleepUninterruptibly(2, TimeUnit.SECONDS); + // when we read with a nowInSeconds before the cell has expired, + // the PartitionIterator includes the row we just wrote + Row row = Util.getOnlyRow(Util.cmd(cfs, dk).includeRow("c1").withNowInSeconds(nowInSeconds).build()); + assertEquals("a1", ByteBufferUtil.string(row.getCell(def).value())); - assert !c.isLive() && c.timestamp() == 0; + // when we read with a nowInSeconds after the cell has expired, the row is filtered + // so the PartitionIterator is empty + Util.assertEmpty(Util.cmd(cfs, dk).includeRow("c1").withNowInSeconds(nowInSeconds + ttl + 1).build()); + } + + private void assertRangeTombstoneMarkers(Slice.Bound start, Slice.Bound end, DeletionTime deletionTime, Object[] expected) + { + AbstractType clusteringType = (AbstractType)cfm.comparator.subtype(0); + + assertEquals(1, start.size()); + assertEquals(start.kind(), Slice.Bound.Kind.INCL_START_BOUND); + assertEquals(expected[0], clusteringType.getString(start.get(0))); + + assertEquals(1, end.size()); + assertEquals(end.kind(), Slice.Bound.Kind.INCL_END_BOUND); + assertEquals(expected[1], clusteringType.getString(end.get(0))); + + assertEquals(expected[2], deletionTime.markedForDeleteAt()); + assertEquals(expected[3], deletionTime.localDeletionTime()); + } + + public void writeRangeTombstone(PartitionUpdate update, Object start, Object end, long markedForDeleteAt, int localDeletionTime) + { + ClusteringComparator comparator = cfs.getComparator(); + update.addRangeTombstone(Slice.make(comparator.make(start), comparator.make(end)), + new SimpleDeletionTime(markedForDeleteAt, localDeletionTime)); + } + + private void writeSimpleCellValue(Row.Writer writer, + CFMetaData cfm, + ColumnDefinition columnDefinition, + String value, + long timestamp, + int nowInSeconds) + { + writer.writeCell(columnDefinition, + false, + ((AbstractType) columnDefinition.cellValueType()).decompose(value), + SimpleLivenessInfo.forUpdate(timestamp, LivenessInfo.NO_TTL, nowInSeconds, cfm), + null); } } diff --git a/test/unit/org/apache/cassandra/db/ScrubTest.java b/test/unit/org/apache/cassandra/db/ScrubTest.java index d4579af2db..5f0e3ac2fd 100644 --- a/test/unit/org/apache/cassandra/db/ScrubTest.java +++ b/test/unit/org/apache/cassandra/db/ScrubTest.java @@ -31,17 +31,22 @@ import java.io.IOError; import java.io.IOException; import java.io.RandomAccessFile; +import org.apache.cassandra.Util; +import org.apache.cassandra.UpdateBuilder; +import org.apache.cassandra.config.ColumnDefinition; import org.apache.cassandra.config.KSMetaData; import org.apache.cassandra.cql3.QueryProcessor; -import org.apache.cassandra.db.composites.CellNameType; +import org.apache.cassandra.db.rows.*; +import org.apache.cassandra.db.partitions.*; import org.apache.cassandra.db.marshal.BytesType; import org.apache.cassandra.db.marshal.CounterColumnType; import org.apache.cassandra.db.marshal.UUIDType; -import org.apache.cassandra.exceptions.ConfigurationException; import org.apache.cassandra.db.compaction.OperationType; import org.apache.cassandra.db.lifecycle.LifecycleTransaction; +import org.apache.cassandra.exceptions.ConfigurationException; import org.apache.cassandra.exceptions.RequestExecutionException; import org.apache.cassandra.io.compress.CompressionMetadata; +import org.apache.cassandra.io.sstable.format.SSTableWriter; import org.apache.cassandra.io.sstable.format.SSTableFormat; import org.apache.cassandra.io.sstable.format.SSTableReader; import org.apache.cassandra.locator.SimpleStrategy; @@ -51,11 +56,9 @@ import org.junit.BeforeClass; import org.junit.Test; import org.junit.runner.RunWith; -import org.apache.cassandra.config.CFMetaData; -import org.apache.cassandra.config.Schema; +import org.apache.cassandra.config.*; import org.apache.cassandra.cql3.Operator; import org.apache.cassandra.cql3.UntypedResultSet; -import org.apache.cassandra.db.columniterator.IdentityQueryFilter; import org.apache.cassandra.db.compaction.CompactionManager; import org.apache.cassandra.db.compaction.Scrubber; import org.apache.cassandra.db.index.SecondaryIndex; @@ -73,11 +76,6 @@ import org.apache.cassandra.utils.ByteBufferUtil; import static org.junit.Assert.*; import static org.junit.Assume.assumeTrue; -import static org.apache.cassandra.Util.cellname; -import static org.apache.cassandra.Util.column; -import static org.junit.Assert.assertEquals; -import static org.junit.Assert.fail; - @RunWith(OrderedJUnit4ClassRunner.class) public class ScrubTest { @@ -90,9 +88,8 @@ public class ScrubTest public static final String CF_INDEX1 = "Indexed1"; public static final String CF_INDEX2 = "Indexed2"; - public static final String COL_KEYS_INDEX = "birthdate"; - public static final String COL_COMPOSITES_INDEX = "col1"; - public static final String COL_NON_INDEX = "notanindexcol"; + public static final String COL_INDEX = "birthdate"; + public static final String COL_NON_INDEX = "notbirthdate"; public static final Integer COMPRESSION_CHUNK_LENGTH = 4096; @@ -106,11 +103,10 @@ public class ScrubTest SchemaLoader.standardCFMD(KEYSPACE, CF), SchemaLoader.standardCFMD(KEYSPACE, CF2), SchemaLoader.standardCFMD(KEYSPACE, CF3), - SchemaLoader.standardCFMD(KEYSPACE, COUNTER_CF) - .defaultValidator(CounterColumnType.instance) + SchemaLoader.counterCFMD(KEYSPACE, COUNTER_CF) .compressionParameters(SchemaLoader.getCompressionParameters(COMPRESSION_CHUNK_LENGTH)), - SchemaLoader.standardCFMD(KEYSPACE, CF_UUID).keyValidator(UUIDType.instance), - SchemaLoader.indexCFMD(KEYSPACE, CF_INDEX1, true), + SchemaLoader.standardCFMD(KEYSPACE, CF_UUID, 0, UUIDType.instance), + SchemaLoader.keysIndexCFMD(KEYSPACE, CF_INDEX1, true), SchemaLoader.compositeIndexCFMD(KEYSPACE, CF_INDEX2, true)); } @@ -122,18 +118,14 @@ public class ScrubTest ColumnFamilyStore cfs = keyspace.getColumnFamilyStore(CF); cfs.clearUnsafe(); - List rows; - // insert data and verify we get it back w/ range query fillCF(cfs, 1); - rows = cfs.getRangeSlice(Util.range("", ""), null, new IdentityQueryFilter(), 1000); - assertEquals(1, rows.size()); + assertOrderedAll(cfs, 1); CompactionManager.instance.performScrub(cfs, false, true); // check data is still there - rows = cfs.getRangeSlice(Util.range("", ""), null, new IdentityQueryFilter(), 1000); - assertEquals(1, rows.size()); + assertOrderedAll(cfs, 1); } @Test @@ -150,8 +142,7 @@ public class ScrubTest fillCounterCF(cfs, numPartitions); - List rows = cfs.getRangeSlice(Util.range("", ""), null, new IdentityQueryFilter(), numPartitions*10); - assertEquals(numPartitions, rows.size()); + assertOrderedAll(cfs, numPartitions); assertEquals(1, cfs.getSSTables().size()); @@ -195,8 +186,7 @@ public class ScrubTest } assertEquals(1, cfs.getSSTables().size()); - rows = cfs.getRangeSlice(Util.range("", ""), null, new IdentityQueryFilter(), 1000); - assertEquals(scrubResult.goodRows, rows.size()); + assertOrderedAll(cfs, scrubResult.goodRows); } @Test @@ -212,8 +202,7 @@ public class ScrubTest fillCounterCF(cfs, 2); - List rows = cfs.getRangeSlice(Util.range("", ""), null, new IdentityQueryFilter(), 1000); - assertEquals(2, rows.size()); + assertOrderedAll(cfs, 2); SSTableReader sstable = cfs.getSSTables().iterator().next(); @@ -240,8 +229,7 @@ public class ScrubTest assertEquals(1, cfs.getSSTables().size()); // verify that we can read all of the rows, and there is now one less row - rows = cfs.getRangeSlice(Util.range("", ""), null, new IdentityQueryFilter(), 1000); - assertEquals(1, rows.size()); + assertOrderedAll(cfs, 1); } @Test @@ -255,12 +243,9 @@ public class ScrubTest ColumnFamilyStore cfs = keyspace.getColumnFamilyStore(CF); cfs.clearUnsafe(); - List rows; - // insert data and verify we get it back w/ range query fillCF(cfs, 4); - rows = cfs.getRangeSlice(Util.range("", ""), null, new IdentityQueryFilter(), 1000); - assertEquals(4, rows.size()); + assertOrderedAll(cfs, 4); SSTableReader sstable = cfs.getSSTables().iterator().next(); overrideWithGarbage(sstable, 0, 2); @@ -268,8 +253,7 @@ public class ScrubTest CompactionManager.instance.performScrub(cfs, false, true); // check data is still there - rows = cfs.getRangeSlice(Util.range("", ""), null, new IdentityQueryFilter(), 1000); - assertEquals(4, rows.size()); + assertOrderedAll(cfs, 4); } @Test @@ -287,24 +271,6 @@ public class ScrubTest } } - @Test - public void testScrubDeletedRow() throws ExecutionException, InterruptedException - { - CompactionManager.instance.disableAutoCompaction(); - Keyspace keyspace = Keyspace.open(KEYSPACE); - ColumnFamilyStore cfs = keyspace.getColumnFamilyStore(CF2); - cfs.clearUnsafe(); - - ColumnFamily cf = ArrayBackedSortedColumns.factory.create(KEYSPACE, CF2); - cf.delete(new DeletionInfo(0, 1)); // expired tombstone - Mutation rm = new Mutation(KEYSPACE, ByteBufferUtil.bytes(1), cf); - rm.applyUnsafe(); - cfs.forceBlockingFlush(); - - CompactionManager.instance.performScrub(cfs, false, true); - assert cfs.getSSTables().isEmpty(); - } - @Test public void testScrubMultiRow() throws ExecutionException, InterruptedException { @@ -313,18 +279,14 @@ public class ScrubTest ColumnFamilyStore cfs = keyspace.getColumnFamilyStore(CF); cfs.clearUnsafe(); - List rows; - // insert data and verify we get it back w/ range query fillCF(cfs, 10); - rows = cfs.getRangeSlice(Util.range("", ""), null, new IdentityQueryFilter(), 1000); - assertEquals(10, rows.size()); + assertOrderedAll(cfs, 10); CompactionManager.instance.performScrub(cfs, false, true); // check data is still there - rows = cfs.getRangeSlice(Util.range("", ""), null, new IdentityQueryFilter(), 1000); - assertEquals(10, rows.size()); + assertOrderedAll(cfs, 10); } @Test @@ -337,24 +299,28 @@ public class ScrubTest cfs.clearUnsafe(); /* - * Code used to generate an outOfOrder sstable. The test for out-of-order key in SSTableWriter must also be commented out. + * Code used to generate an outOfOrder sstable. The test for out-of-order key in BigTableWriter must also be commented out. * The test also assumes an ordered partitioner. - * - ColumnFamily cf = ArrayBackedSortedColumns.factory.create(cfs.metadata); - cf.addColumn(new BufferCell(ByteBufferUtil.bytes("someName"), ByteBufferUtil.bytes("someValue"), 0L)); + List keys = Arrays.asList("t", "a", "b", "z", "c", "y", "d"); + String filename = cfs.getTempSSTablePath(new File(System.getProperty("corrupt-sstable-root"))); + SSTableWriter writer = SSTableWriter.create(cfs.metadata, + Descriptor.fromFilename(filename), + keys.size(), + 0L, + 0, + DatabaseDescriptor.getPartitioner(), + SerializationHeader.make(cfs.metadata, Collections.emptyList())); - SSTableWriter writer = new SSTableWriter(cfs.getTempSSTablePath(new File(System.getProperty("corrupt-sstable-root"))), - cfs.metadata.getIndexInterval(), - cfs.metadata, - cfs.partitioner, - SSTableMetadata.createCollector(BytesType.instance)); - writer.append(Util.dk("a"), cf); - writer.append(Util.dk("b"), cf); - writer.append(Util.dk("z"), cf); - writer.append(Util.dk("c"), cf); - writer.append(Util.dk("y"), cf); - writer.append(Util.dk("d"), cf); - writer.finish(); + for (String k : keys) + { + RowUpdateBuilder builder = new RowUpdateBuilder(cfs.metadata, 0L, Util.dk(k)); + PartitionUpdate update = builder.clustering("someName") + .add("val", "someValue") + .buildUpdate(); + + writer.append(update.unfilteredIterator()); + } + writer.finish(false); */ String root = System.getProperty("corrupt-sstable-root"); @@ -362,7 +328,7 @@ public class ScrubTest File rootDir = new File(root); assert rootDir.isDirectory(); - Descriptor desc = new Descriptor("jb", rootDir, KEYSPACE, columnFamily, 1, Descriptor.Type.FINAL, SSTableFormat.Type.LEGACY); + Descriptor desc = new Descriptor("la", rootDir, KEYSPACE, columnFamily, 1, Descriptor.Type.FINAL, SSTableFormat.Type.BIG); CFMetaData metadata = Schema.instance.getCFMetaData(desc.ksname, desc.cfname); try @@ -374,7 +340,7 @@ public class ScrubTest // open without validation for scrubbing Set components = new HashSet<>(); - components.add(Component.COMPRESSION_INFO); + //components.add(Component.COMPRESSION_INFO); components.add(Component.DATA); components.add(Component.PRIMARY_INDEX); components.add(Component.FILTER); @@ -392,9 +358,8 @@ public class ScrubTest scrubber.scrub(); } cfs.loadNewSSTables(); - List rows = cfs.getRangeSlice(Util.range("", ""), null, new IdentityQueryFilter(), 1000); - assert isRowOrdered(rows) : "Scrub failed: " + rows; - assert rows.size() == 6 : "Got " + rows.size(); + assertOrderedAll(cfs, 7); + sstable.selfRef().release(); } private void overrideWithGarbage(SSTableReader sstable, ByteBuffer key1, ByteBuffer key2) throws IOException @@ -407,17 +372,19 @@ public class ScrubTest CompressionMetadata compData = CompressionMetadata.create(sstable.getFilename()); CompressionMetadata.Chunk chunk1 = compData.chunkFor( - sstable.getPosition(RowPosition.ForKey.get(key1, sstable.partitioner), SSTableReader.Operator.EQ).position); + sstable.getPosition(PartitionPosition.ForKey.get(key1, sstable.partitioner), SSTableReader.Operator.EQ).position); CompressionMetadata.Chunk chunk2 = compData.chunkFor( - sstable.getPosition(RowPosition.ForKey.get(key2, sstable.partitioner), SSTableReader.Operator.EQ).position); + sstable.getPosition(PartitionPosition.ForKey.get(key2, sstable.partitioner), SSTableReader.Operator.EQ).position); startPosition = Math.min(chunk1.offset, chunk2.offset); endPosition = Math.max(chunk1.offset + chunk1.length, chunk2.offset + chunk2.length); + + compData.close(); } else { // overwrite with garbage from key1 to key2 - long row0Start = sstable.getPosition(RowPosition.ForKey.get(key1, sstable.partitioner), SSTableReader.Operator.EQ).position; - long row1Start = sstable.getPosition(RowPosition.ForKey.get(key2, sstable.partitioner), SSTableReader.Operator.EQ).position; + long row0Start = sstable.getPosition(PartitionPosition.ForKey.get(key1, sstable.partitioner), SSTableReader.Operator.EQ).position; + long row1Start = sstable.getPosition(PartitionPosition.ForKey.get(key2, sstable.partitioner), SSTableReader.Operator.EQ).position; startPosition = Math.min(row0Start, row1Start); endPosition = Math.max(row0Start, row1Start); } @@ -433,28 +400,35 @@ public class ScrubTest file.close(); } - private static boolean isRowOrdered(List rows) + private static void assertOrderedAll(ColumnFamilyStore cfs, int expectedSize) { - DecoratedKey prev = null; - for (Row row : rows) - { - if (prev != null && prev.compareTo(row.key) > 0) - return false; - prev = row.key; - } - return true; + assertOrdered(Util.cmd(cfs).build(), expectedSize); } - protected void fillCF(ColumnFamilyStore cfs, int rowsPerSSTable) + private static void assertOrdered(ReadCommand cmd, int expectedSize) { - for (int i = 0; i < rowsPerSSTable; i++) + int size = 0; + DecoratedKey prev = null; + for (Partition partition : Util.getAllUnfiltered(cmd)) { - String key = String.valueOf(i); - ColumnFamily cf = ArrayBackedSortedColumns.factory.create(KEYSPACE, CF); - cf.addColumn(column("c1", "1", 1L)); - cf.addColumn(column("c2", "2", 1L)); - Mutation rm = new Mutation(KEYSPACE, ByteBufferUtil.bytes(key), cf); - rm.applyUnsafe(); + DecoratedKey current = partition.partitionKey(); + assertTrue("key " + current + " does not sort after previous key " + prev, prev == null || prev.compareTo(current) < 0); + prev = current; + ++size; + } + assertEquals(expectedSize, size); + } + + protected void fillCF(ColumnFamilyStore cfs, int partitionsPerSSTable) + { + for (int i = 0; i < partitionsPerSSTable; i++) + { + PartitionUpdate update = UpdateBuilder.create(cfs.metadata, String.valueOf(i)) + .newRow("r1").add("val", "1") + .newRow("r1").add("val", "1") + .build(); + + new Mutation(update).applyUnsafe(); } cfs.forceBlockingFlush(); @@ -465,36 +439,33 @@ public class ScrubTest assertTrue(values.length % 2 == 0); for (int i = 0; i < values.length; i +=2) { - String key = String.valueOf(i); - ColumnFamily cf = ArrayBackedSortedColumns.factory.create(KEYSPACE, cfs.name); + UpdateBuilder builder = UpdateBuilder.create(cfs.metadata, String.valueOf(i)); if (composite) { - String clusterKey = "c" + key; - cf.addColumn(column(clusterKey, COL_COMPOSITES_INDEX, values[i], 1L)); - cf.addColumn(column(clusterKey, COL_NON_INDEX, values[i + 1], 1L)); + builder.newRow("c" + i) + .add(COL_INDEX, values[i]) + .add(COL_NON_INDEX, values[i + 1]); } else { - cf.addColumn(column(COL_KEYS_INDEX, values[i], 1L)); - cf.addColumn(column(COL_NON_INDEX, values[i + 1], 1L)); + builder.newRow() + .add(COL_INDEX, values[i]) + .add(COL_NON_INDEX, values[i + 1]); } - Mutation rm = new Mutation(KEYSPACE, ByteBufferUtil.bytes(key), cf); - rm.applyUnsafe(); + new Mutation(builder.build()).applyUnsafe(); } cfs.forceBlockingFlush(); } - protected void fillCounterCF(ColumnFamilyStore cfs, int rowsPerSSTable) throws WriteTimeoutException + protected void fillCounterCF(ColumnFamilyStore cfs, int partitionsPerSSTable) throws WriteTimeoutException { - for (int i = 0; i < rowsPerSSTable; i++) + for (int i = 0; i < partitionsPerSSTable; i++) { - String key = String.valueOf(i); - ColumnFamily cf = ArrayBackedSortedColumns.factory.create(KEYSPACE, COUNTER_CF); - Mutation rm = new Mutation(KEYSPACE, ByteBufferUtil.bytes(key), cf); - rm.addCounter(COUNTER_CF, cellname("Column1"), 100); - CounterMutation cm = new CounterMutation(rm, ConsistencyLevel.ONE); - cm.apply(); + PartitionUpdate update = UpdateBuilder.create(cfs.metadata, String.valueOf(i)) + .newRow("r1").add("val", 100L) + .build(); + new CounterMutation(new Mutation(update), ConsistencyLevel.ONE).apply(); } cfs.forceBlockingFlush(); @@ -514,34 +485,13 @@ public class ScrubTest QueryProcessor.process("CREATE TABLE \"Keyspace1\".test_scrub_validation (a text primary key, b int)", ConsistencyLevel.ONE); ColumnFamilyStore cfs2 = keyspace.getColumnFamilyStore("test_scrub_validation"); - Mutation mutation = new Mutation("Keyspace1", UTF8Type.instance.decompose("key")); - CellNameType ct = cfs2.getComparator(); - mutation.add("test_scrub_validation", ct.makeCellName("b"), LongType.instance.decompose(1L), System.currentTimeMillis()); - mutation.apply(); + + new Mutation(UpdateBuilder.create(cfs2.metadata, "key").newRow().add("b", LongType.instance.decompose(1L)).build()).apply(); cfs2.forceBlockingFlush(); CompactionManager.instance.performScrub(cfs2, false, false); } - /** - * Tests CASSANDRA-6892 (key aliases being used improperly for validation) - */ - @Test - public void testColumnNameEqualToDefaultKeyAlias() throws ExecutionException, InterruptedException - { - Keyspace keyspace = Keyspace.open(KEYSPACE); - ColumnFamilyStore cfs = keyspace.getColumnFamilyStore(CF_UUID); - - ColumnFamily cf = ArrayBackedSortedColumns.factory.create(KEYSPACE, CF_UUID); - cf.addColumn(column(CFMetaData.DEFAULT_KEY_ALIAS, "not a uuid", 1L)); - Mutation mutation = new Mutation(KEYSPACE, ByteBufferUtil.bytes(UUIDGen.getTimeUUID()), cf); - mutation.applyUnsafe(); - cfs.forceBlockingFlush(); - CompactionManager.instance.performScrub(cfs, false, true); - - assertEquals(1, cfs.getSSTables().size()); - } - /** * For CASSANDRA-6892 too, check that for a compact table with one cluster column, we can insert whatever * we want as value for the clustering column, including something that would conflict with a CQL column definition. @@ -576,46 +526,46 @@ public class ScrubTest //If the partitioner preserves the order then SecondaryIndex uses BytesType comparator, // otherwise it uses LocalByPartitionerType setKeyComparator(BytesType.instance); - testScrubIndex(CF_INDEX1, COL_KEYS_INDEX, false, true); + testScrubIndex(CF_INDEX1, COL_INDEX, false, true); } @Test /* CASSANDRA-5174 */ public void testScrubCompositeIndex_preserveOrder() throws IOException, ExecutionException, InterruptedException { setKeyComparator(BytesType.instance); - testScrubIndex(CF_INDEX2, COL_COMPOSITES_INDEX, true, true); + testScrubIndex(CF_INDEX2, COL_INDEX, true, true); } @Test /* CASSANDRA-5174 */ public void testScrubKeysIndex() throws IOException, ExecutionException, InterruptedException { setKeyComparator(new LocalByPartionerType(StorageService.getPartitioner())); - testScrubIndex(CF_INDEX1, COL_KEYS_INDEX, false, true); + testScrubIndex(CF_INDEX1, COL_INDEX, false, true); } @Test /* CASSANDRA-5174 */ public void testScrubCompositeIndex() throws IOException, ExecutionException, InterruptedException { setKeyComparator(new LocalByPartionerType(StorageService.getPartitioner())); - testScrubIndex(CF_INDEX2, COL_COMPOSITES_INDEX, true, true); + testScrubIndex(CF_INDEX2, COL_INDEX, true, true); } @Test /* CASSANDRA-5174 */ public void testFailScrubKeysIndex() throws IOException, ExecutionException, InterruptedException { - testScrubIndex(CF_INDEX1, COL_KEYS_INDEX, false, false); + testScrubIndex(CF_INDEX1, COL_INDEX, false, false); } @Test /* CASSANDRA-5174 */ public void testFailScrubCompositeIndex() throws IOException, ExecutionException, InterruptedException { - testScrubIndex(CF_INDEX2, COL_COMPOSITES_INDEX, true, false); + testScrubIndex(CF_INDEX2, COL_INDEX, true, false); } @Test /* CASSANDRA-5174 */ public void testScrubTwice() throws IOException, ExecutionException, InterruptedException { - testScrubIndex(CF_INDEX1, COL_KEYS_INDEX, false, true, true); + testScrubIndex(CF_INDEX1, COL_INDEX, false, true, true); } /** The SecondaryIndex class is used for custom indexes so to avoid @@ -663,10 +613,8 @@ public class ScrubTest fillIndexCF(cfs, composite, colValues); // check index - IndexExpression expr = new IndexExpression(ByteBufferUtil.bytes(colName), Operator.EQ, ByteBufferUtil.bytes(1L)); - List rows = cfs.search(Util.range("", ""), Arrays.asList(expr), new IdentityQueryFilter(), numRows); - assertNotNull(rows); - assertEquals(numRows / 2, rows.size()); + + assertOrdered(Util.cmd(cfs).filterOn(colName, Operator.EQ, 1L).build(), numRows / 2); // scrub index Set indexCfss = cfs.indexManager.getIndexesBackedByCfs(); @@ -690,8 +638,6 @@ public class ScrubTest // check index is still working - rows = cfs.search(Util.range("", ""), Arrays.asList(expr), new IdentityQueryFilter(), numRows); - assertNotNull(rows); - assertEquals(numRows / 2, rows.size()); + assertOrdered(Util.cmd(cfs).filterOn(colName, Operator.EQ, 1L).build(), numRows / 2); } } diff --git a/test/unit/org/apache/cassandra/db/SecondaryIndexTest.java b/test/unit/org/apache/cassandra/db/SecondaryIndexTest.java new file mode 100644 index 0000000000..c6372b2cd3 --- /dev/null +++ b/test/unit/org/apache/cassandra/db/SecondaryIndexTest.java @@ -0,0 +1,490 @@ +/* +* Licensed to the Apache Software Foundation (ASF) under one +* or more contributor license agreements. See the NOTICE file +* distributed with this work for additional information +* regarding copyright ownership. The ASF licenses this file +* to you under the Apache License, Version 2.0 (the +* "License"); you may not use this file except in compliance +* with the License. You may obtain a copy of the License at +* +* http://www.apache.org/licenses/LICENSE-2.0 +* +* Unless required by applicable law or agreed to in writing, +* software distributed under the License is distributed on an +* "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +* KIND, either express or implied. See the License for the +* specific language governing permissions and limitations +* under the License. +*/ +package org.apache.cassandra.db; + +import java.io.IOException; +import java.nio.ByteBuffer; +import java.util.*; +import java.util.concurrent.ExecutionException; +import java.util.concurrent.Future; +import java.util.concurrent.TimeUnit; + +import com.google.common.collect.Iterators; +import org.junit.Before; +import org.junit.BeforeClass; +import org.junit.Test; + +import org.apache.cassandra.SchemaLoader; +import org.apache.cassandra.Util; +import org.apache.cassandra.config.ColumnDefinition; +import org.apache.cassandra.config.IndexType; +import org.apache.cassandra.config.KSMetaData; +import org.apache.cassandra.cql3.Operator; +import org.apache.cassandra.db.index.SecondaryIndex; +import org.apache.cassandra.db.index.SecondaryIndexSearcher; +import org.apache.cassandra.db.marshal.AbstractType; +import org.apache.cassandra.db.partitions.*; +import org.apache.cassandra.db.rows.Row; +import org.apache.cassandra.db.rows.RowIterator; +import org.apache.cassandra.exceptions.ConfigurationException; +import org.apache.cassandra.locator.SimpleStrategy; +import org.apache.cassandra.utils.ByteBufferUtil; +import org.apache.cassandra.utils.FBUtilities; + +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertFalse; +import static org.junit.Assert.assertTrue; + +public class SecondaryIndexTest +{ + public static final String KEYSPACE1 = "SecondaryIndexTest1"; + public static final String WITH_COMPOSITE_INDEX = "WithCompositeIndex"; + public static final String WITH_KEYS_INDEX = "WithKeysIndex"; + public static final String COMPOSITE_INDEX_TO_BE_ADDED = "CompositeIndexToBeAdded"; + + @BeforeClass + public static void defineSchema() throws ConfigurationException + { + SchemaLoader.prepareServer(); + SchemaLoader.createKeyspace(KEYSPACE1, + SimpleStrategy.class, + KSMetaData.optsWithRF(1), + SchemaLoader.compositeIndexCFMD(KEYSPACE1, WITH_COMPOSITE_INDEX, true).gcGraceSeconds(0), + SchemaLoader.compositeIndexCFMD(KEYSPACE1, COMPOSITE_INDEX_TO_BE_ADDED, false).gcGraceSeconds(0), + SchemaLoader.keysIndexCFMD(KEYSPACE1, WITH_KEYS_INDEX, true).gcGraceSeconds(0)); + } + + @Before + public void truncateCFS() + { + Keyspace.open(KEYSPACE1).getColumnFamilyStore(WITH_COMPOSITE_INDEX).truncateBlocking(); + Keyspace.open(KEYSPACE1).getColumnFamilyStore(COMPOSITE_INDEX_TO_BE_ADDED).truncateBlocking(); + Keyspace.open(KEYSPACE1).getColumnFamilyStore(WITH_KEYS_INDEX).truncateBlocking(); + } + + @Test + public void testIndexScan() + { + ColumnFamilyStore cfs = Keyspace.open(KEYSPACE1).getColumnFamilyStore(WITH_COMPOSITE_INDEX); + + new RowUpdateBuilder(cfs.metadata, 0, "k1").clustering("c").add("birthdate", 1L).add("notbirthdate", 1L).build().applyUnsafe(); + new RowUpdateBuilder(cfs.metadata, 0, "k2").clustering("c").add("birthdate", 2L).add("notbirthdate", 2L).build().applyUnsafe(); + new RowUpdateBuilder(cfs.metadata, 0, "k3").clustering("c").add("birthdate", 1L).add("notbirthdate", 2L).build().applyUnsafe(); + new RowUpdateBuilder(cfs.metadata, 0, "k4").clustering("c").add("birthdate", 3L).add("notbirthdate", 2L).build().applyUnsafe(); + + // basic single-expression query + List partitions = Util.getAll(Util.cmd(cfs).fromKeyExcl("k1").toKeyIncl("k3").columns("birthdate").build()); + assertEquals(2, partitions.size()); + Util.assertCellValue(2L, cfs, Util.row(partitions.get(0), "c"), "birthdate"); + Util.assertCellValue(1L, cfs, Util.row(partitions.get(1), "c"), "birthdate"); + + // 2 columns, 3 results + partitions = Util.getAll(Util.cmd(cfs).fromKeyExcl("k1").toKeyIncl("k4aaa").build()); + assertEquals(3, partitions.size()); + + Row first = Util.row(partitions.get(0), "c"); + Util.assertCellValue(2L, cfs, first, "birthdate"); + Util.assertCellValue(2L, cfs, first, "notbirthdate"); + + Row second = Util.row(partitions.get(1), "c"); + Util.assertCellValue(1L, cfs, second, "birthdate"); + Util.assertCellValue(2L, cfs, second, "notbirthdate"); + + Row third = Util.row(partitions.get(2), "c"); + Util.assertCellValue(3L, cfs, third, "birthdate"); + Util.assertCellValue(2L, cfs, third, "notbirthdate"); + + // Verify getIndexSearchers finds the data for our rc + ReadCommand rc = Util.cmd(cfs).fromKeyIncl("k1") + .toKeyIncl("k3") + .columns("birthdate") + .filterOn("birthdate", Operator.EQ, 1L) + .build(); + List searchers = cfs.indexManager.getIndexSearchersFor(rc); + assertEquals(searchers.size(), 1); + try (ReadOrderGroup orderGroup = rc.startOrderGroup(); UnfilteredPartitionIterator pi = searchers.get(0).search(rc, orderGroup)) + { + assertTrue(pi.hasNext()); + pi.next().close(); + } + + // Verify gt on idx scan + partitions = Util.getAll(Util.cmd(cfs).fromKeyIncl("k1").toKeyIncl("k4aaa") .filterOn("birthdate", Operator.GT, 1L).build()); + int rowCount = 0; + for (FilteredPartition partition : partitions) + { + for (Row row : partition) + { + ++rowCount; + assert ByteBufferUtil.toLong(Util.cell(cfs, row, "birthdate").value()) > 1L; + } + } + assertEquals(2, rowCount); + + // Filter on non-indexed, LT comparison + Util.assertEmpty(Util.cmd(cfs).fromKeyExcl("k1").toKeyIncl("k4aaa") + .filterOn("notbirthdate", Operator.NEQ, 2L) + .build()); + + // Hit on primary, fail on non-indexed filter + Util.assertEmpty(Util.cmd(cfs).fromKeyExcl("k1").toKeyIncl("k4aaa") + .filterOn("birthdate", Operator.EQ, 1L) + .filterOn("notbirthdate", Operator.NEQ, 2L) + .build()); + } + + @Test + public void testLargeScan() + { + ColumnFamilyStore cfs = Keyspace.open(KEYSPACE1).getColumnFamilyStore(WITH_COMPOSITE_INDEX); + ByteBuffer bBB = ByteBufferUtil.bytes("birthdate"); + ByteBuffer nbBB = ByteBufferUtil.bytes("notbirthdate"); + + for (int i = 0; i < 100; i++) + { + new RowUpdateBuilder(cfs.metadata, FBUtilities.timestampMicros(), "key" + i) + .clustering("c") + .add("birthdate", 34L) + .add("notbirthdate", ByteBufferUtil.bytes((long) (i % 2))) + .build() + .applyUnsafe(); + } + + List partitions = Util.getAll(Util.cmd(cfs) + .filterOn("birthdate", Operator.EQ, 34L) + .filterOn("notbirthdate", Operator.EQ, 1L) + .build()); + + Set keys = new HashSet<>(); + int rowCount = 0; + + for (FilteredPartition partition : partitions) + { + keys.add(partition.partitionKey()); + rowCount += partition.rowCount(); + } + + // extra check that there are no duplicate results -- see https://issues.apache.org/jira/browse/CASSANDRA-2406 + assertEquals(rowCount, keys.size()); + assertEquals(50, rowCount); + } + + @Test + public void testCompositeIndexDeletions() throws IOException + { + ColumnFamilyStore cfs = Keyspace.open(KEYSPACE1).getColumnFamilyStore(WITH_COMPOSITE_INDEX); + ByteBuffer bBB = ByteBufferUtil.bytes("birthdate"); + ColumnDefinition bDef = cfs.metadata.getColumnDefinition(bBB); + ByteBuffer col = ByteBufferUtil.bytes("birthdate"); + + // Confirm addition works + new RowUpdateBuilder(cfs.metadata, 0, "k1").clustering("c").add("birthdate", 1L).build().applyUnsafe(); + assertIndexedOne(cfs, col, 1L); + + // delete the column directly + RowUpdateBuilder.deleteRow(cfs.metadata, 1, "k1", "c").applyUnsafe(); + assertIndexedNone(cfs, col, 1L); + + // verify that it's not being indexed under any other value either + ReadCommand rc = Util.cmd(cfs).build(); + assertEquals(0, cfs.indexManager.getIndexSearchersFor(rc).size()); + + // resurrect w/ a newer timestamp + new RowUpdateBuilder(cfs.metadata, 2, "k1").clustering("c").add("birthdate", 1L).build().apply();; + assertIndexedOne(cfs, col, 1L); + + // verify that row and delete w/ older timestamp does nothing + RowUpdateBuilder.deleteRow(cfs.metadata, 1, "k1", "c").applyUnsafe(); + assertIndexedOne(cfs, col, 1L); + + // similarly, column delete w/ older timestamp should do nothing + new RowUpdateBuilder(cfs.metadata, 1, "k1").clustering("c").delete(bDef).build().applyUnsafe(); + assertIndexedOne(cfs, col, 1L); + + // delete the entire row (w/ newer timestamp this time) + RowUpdateBuilder.deleteRow(cfs.metadata, 3, "k1", "c").applyUnsafe(); + rc = Util.cmd(cfs).build(); + assertEquals(0, cfs.indexManager.getIndexSearchersFor(rc).size()); + + // make sure obsolete mutations don't generate an index entry + new RowUpdateBuilder(cfs.metadata, 3, "k1").clustering("c").add("birthdate", 1L).build().apply();; + rc = Util.cmd(cfs).build(); + assertEquals(0, cfs.indexManager.getIndexSearchersFor(rc).size()); + } + + @Test + public void testCompositeIndexUpdate() throws IOException + { + Keyspace keyspace = Keyspace.open(KEYSPACE1); + ColumnFamilyStore cfs = keyspace.getColumnFamilyStore(WITH_COMPOSITE_INDEX); + ByteBuffer col = ByteBufferUtil.bytes("birthdate"); + + // create a row and update the birthdate value, test that the index query fetches the new version + new RowUpdateBuilder(cfs.metadata, 1, "testIndexUpdate").clustering("c").add("birthdate", 100L).build().applyUnsafe(); + new RowUpdateBuilder(cfs.metadata, 2, "testIndexUpdate").clustering("c").add("birthdate", 200L).build().applyUnsafe(); + + // Confirm old version fetch fails + assertIndexedNone(cfs, col, 100L); + + // Confirm new works + assertIndexedOne(cfs, col, 200L); + + // update the birthdate value with an OLDER timestamp, and test that the index ignores this + assertIndexedNone(cfs, col, 300L); + assertIndexedOne(cfs, col, 200L); + } + + @Test + public void testIndexUpdateOverwritingExpiringColumns() throws Exception + { + // see CASSANDRA-7268 + ColumnFamilyStore cfs = Keyspace.open(KEYSPACE1).getColumnFamilyStore(WITH_COMPOSITE_INDEX); + ByteBuffer col = ByteBufferUtil.bytes("birthdate"); + + // create a row and update the birthdate value with an expiring column + new RowUpdateBuilder(cfs.metadata, 1L, 500, "K100").clustering("c").add("birthdate", 100L).build().applyUnsafe(); + assertIndexedOne(cfs, col, 100L); + + // requires a 1s sleep because we calculate local expiry time as (now() / 1000) + ttl + TimeUnit.SECONDS.sleep(1); + + // now overwrite with the same name/value/ttl, but the local expiry time will be different + new RowUpdateBuilder(cfs.metadata, 1L, 500, "K100").clustering("c").add("birthdate", 100L).build().applyUnsafe(); + assertIndexedOne(cfs, col, 100L); + + // check that modifying the indexed value using the same timestamp behaves as expected + new RowUpdateBuilder(cfs.metadata, 1L, 500, "K101").clustering("c").add("birthdate", 101L).build().applyUnsafe(); + assertIndexedOne(cfs, col, 101L); + + TimeUnit.SECONDS.sleep(1); + + new RowUpdateBuilder(cfs.metadata, 1L, 500, "K101").clustering("c").add("birthdate", 102L).build().applyUnsafe(); + // Confirm 101 is gone + assertIndexedNone(cfs, col, 101L); + + // Confirm 102 is there + assertIndexedOne(cfs, col, 102L); + } + + @Test + public void testDeleteOfInconsistentValuesInKeysIndex() throws Exception + { + Keyspace keyspace = Keyspace.open(KEYSPACE1); + ColumnFamilyStore cfs = keyspace.getColumnFamilyStore(WITH_KEYS_INDEX); + + ByteBuffer col = ByteBufferUtil.bytes("birthdate"); + + // create a row and update the "birthdate" value + new RowUpdateBuilder(cfs.metadata, 1, "k1").noRowMarker().add("birthdate", 1L).build().applyUnsafe(); + + // force a flush, so our index isn't being read from a memtable + keyspace.getColumnFamilyStore(WITH_KEYS_INDEX).forceBlockingFlush(); + + // now apply another update, but force the index update to be skipped + keyspace.apply(new RowUpdateBuilder(cfs.metadata, 2, "k1").noRowMarker().add("birthdate", 2L).build(), true, false); + + // Now searching the index for either the old or new value should return 0 rows + // because the new value was not indexed and the old value should be ignored + // (and in fact purged from the index cf). + // first check for the old value + assertIndexedNone(cfs, col, 1L); + assertIndexedNone(cfs, col, 2L); + + // now, reset back to the original value, still skipping the index update, to + // make sure the value was expunged from the index when it was discovered to be inconsistent + keyspace.apply(new RowUpdateBuilder(cfs.metadata, 3, "k1").noRowMarker().add("birthdate", 1L).build(), true, false); + assertIndexedNone(cfs, col, 1L); + } + + @Test + public void testDeleteOfInconsistentValuesFromCompositeIndex() throws Exception + { + Keyspace keyspace = Keyspace.open(KEYSPACE1); + String cfName = WITH_COMPOSITE_INDEX; + + ColumnFamilyStore cfs = keyspace.getColumnFamilyStore(cfName); + + ByteBuffer col = ByteBufferUtil.bytes("birthdate"); + + // create a row and update the author value + new RowUpdateBuilder(cfs.metadata, 0, "k1").clustering("c").add("birthdate", 10l).build().applyUnsafe(); + + // test that the index query fetches this version + assertIndexedOne(cfs, col, 10l); + + // force a flush and retry the query, so our index isn't being read from a memtable + keyspace.getColumnFamilyStore(cfName).forceBlockingFlush(); + assertIndexedOne(cfs, col, 10l); + + // now apply another update, but force the index update to be skipped + keyspace.apply(new RowUpdateBuilder(cfs.metadata, 1, "k1").clustering("c").add("birthdate", 20l).build(), true, false); + + // Now searching the index for either the old or new value should return 0 rows + // because the new value was not indexed and the old value should be ignored + // (and in fact purged from the index cf). + // first check for the old value + assertIndexedNone(cfs, col, 10l); + assertIndexedNone(cfs, col, 20l); + + // now, reset back to the original value, still skipping the index update, to + // make sure the value was expunged from the index when it was discovered to be inconsistent + // TODO: Figure out why this is re-inserting + keyspace.apply(new RowUpdateBuilder(cfs.metadata, 2, "k1").clustering("c1").add("birthdate", 10l).build(), true, false); + assertIndexedNone(cfs, col, 20l); + } + + // See CASSANDRA-6098 + @Test + public void testDeleteCompositeIndex() throws Exception + { + ColumnFamilyStore cfs = Keyspace.open(KEYSPACE1).getColumnFamilyStore(WITH_COMPOSITE_INDEX); + + ByteBuffer colName = ByteBufferUtil.bytes("birthdate"); + + // Insert indexed value. + new RowUpdateBuilder(cfs.metadata, 1, "k1").clustering("c").add("birthdate", 10l).build().applyUnsafe(); + + // Now delete the value + RowUpdateBuilder.deleteRow(cfs.metadata, 2, "k1", "c").applyUnsafe(); + + // We want the data to be gcable, but even if gcGrace == 0, we still need to wait 1 second + // since we won't gc on a tie. + try { Thread.sleep(1000); } catch (Exception e) {} + + // Read the index and we check we do get no value (and no NPE) + // Note: the index will return the entry because it hasn't been deleted (we + // haven't read yet nor compacted) but the data read itself will return null + assertIndexedNone(cfs, colName, 10l); + } + + @Test + public void testDeleteKeysIndex() throws Exception + { + ColumnFamilyStore cfs = Keyspace.open(KEYSPACE1).getColumnFamilyStore(WITH_KEYS_INDEX); + + ByteBuffer colName = ByteBufferUtil.bytes("birthdate"); + + // Insert indexed value. + new RowUpdateBuilder(cfs.metadata, 1, "k1").add("birthdate", 10l).build().applyUnsafe(); + + // Now delete the value + RowUpdateBuilder.deleteRow(cfs.metadata, 2, "k1").applyUnsafe(); + + // We want the data to be gcable, but even if gcGrace == 0, we still need to wait 1 second + // since we won't gc on a tie. + try { Thread.sleep(1000); } catch (Exception e) {} + + // Read the index and we check we do get no value (and no NPE) + // Note: the index will return the entry because it hasn't been deleted (we + // haven't read yet nor compacted) but the data read itself will return null + assertIndexedNone(cfs, colName, 10l); + } + + // See CASSANDRA-2628 + @Test + public void testIndexScanWithLimitOne() + { + ColumnFamilyStore cfs = Keyspace.open(KEYSPACE1).getColumnFamilyStore(WITH_COMPOSITE_INDEX); + Mutation rm; + + new RowUpdateBuilder(cfs.metadata, 0, "kk1").clustering("c").add("birthdate", 1L).build().applyUnsafe(); + new RowUpdateBuilder(cfs.metadata, 0, "kk1").clustering("c").add("notbirthdate", 1L).build().applyUnsafe(); + new RowUpdateBuilder(cfs.metadata, 0, "kk2").clustering("c").add("birthdate", 1L).build().applyUnsafe(); + new RowUpdateBuilder(cfs.metadata, 0, "kk2").clustering("c").add("notbirthdate", 2L).build().applyUnsafe(); + new RowUpdateBuilder(cfs.metadata, 0, "kk3").clustering("c").add("birthdate", 1L).build().applyUnsafe(); + new RowUpdateBuilder(cfs.metadata, 0, "kk3").clustering("c").add("notbirthdate", 2L).build().applyUnsafe(); + new RowUpdateBuilder(cfs.metadata, 0, "kk4").clustering("c").add("birthdate", 1L).build().applyUnsafe(); + new RowUpdateBuilder(cfs.metadata, 0, "kk4").clustering("c").add("notbirthdate", 2L).build().applyUnsafe(); + + // basic single-expression query, limit 1 + Util.getOnlyRow(Util.cmd(cfs) + .filterOn("birthdate", Operator.EQ, 1L) + .filterOn("notbirthdate", Operator.EQ, 1L) + .withLimit(1) + .build()); + } + + @Test + public void testIndexCreate() throws IOException, InterruptedException, ExecutionException + { + Keyspace keyspace = Keyspace.open(KEYSPACE1); + ColumnFamilyStore cfs = keyspace.getColumnFamilyStore(COMPOSITE_INDEX_TO_BE_ADDED); + + // create a row and update the birthdate value, test that the index query fetches the new version + DecoratedKey dk = Util.dk("k1"); + new RowUpdateBuilder(cfs.metadata, 0, "k1").clustering("c").add("birthdate", 1L).build().applyUnsafe(); + + ColumnDefinition old = cfs.metadata.getColumnDefinition(ByteBufferUtil.bytes("birthdate")); + old.setIndex("birthdate_index", IndexType.COMPOSITES, Collections.EMPTY_MAP); + Future future = cfs.indexManager.addIndexedColumn(old); + future.get(); + // we had a bug (CASSANDRA-2244) where index would get created but not flushed -- check for that + ColumnDefinition cDef = cfs.metadata.getColumnDefinition(ByteBufferUtil.bytes("birthdate")); + assert cfs.indexManager.getIndexForColumn(cDef).getIndexCfs().getSSTables().size() > 0; + + assertIndexedOne(cfs, ByteBufferUtil.bytes("birthdate"), 1L); + + // validate that drop clears it out & rebuild works (CASSANDRA-2320) + SecondaryIndex indexedCfs = cfs.indexManager.getIndexForColumn(cDef); + cfs.indexManager.removeIndexedColumn(ByteBufferUtil.bytes("birthdate")); + assert !indexedCfs.isIndexBuilt(ByteBufferUtil.bytes("birthdate")); + + // rebuild & re-query + future = cfs.indexManager.addIndexedColumn(cDef); + future.get(); + assertIndexedOne(cfs, ByteBufferUtil.bytes("birthdate"), 1L); + } + + @Test + public void testKeysSearcherSimple() throws Exception + { + // Create secondary index and flush to disk + Keyspace keyspace = Keyspace.open(KEYSPACE1); + ColumnFamilyStore cfs = keyspace.getColumnFamilyStore(WITH_KEYS_INDEX); + + for (int i = 0; i < 10; i++) + new RowUpdateBuilder(cfs.metadata, 0, "k" + i).noRowMarker().add("birthdate", 1l).build().applyUnsafe(); + + cfs.forceBlockingFlush(); + assertIndexedCount(cfs, ByteBufferUtil.bytes("birthdate"), 1l, 10); + } + + private void assertIndexedNone(ColumnFamilyStore cfs, ByteBuffer col, Object val) + { + assertIndexedCount(cfs, col, val, 0); + } + private void assertIndexedOne(ColumnFamilyStore cfs, ByteBuffer col, Object val) + { + assertIndexedCount(cfs, col, val, 1); + } + private void assertIndexedCount(ColumnFamilyStore cfs, ByteBuffer col, Object val, int count) + { + ColumnDefinition cdef = cfs.metadata.getColumnDefinition(col); + + ReadCommand rc = Util.cmd(cfs).filterOn(cdef.name.toString(), Operator.EQ, ((AbstractType) cdef.cellValueType()).decompose(val)).build(); + List searchers = cfs.indexManager.getIndexSearchersFor(rc); + if (count != 0) + assertTrue(searchers.size() > 0); + + try (ReadOrderGroup orderGroup = rc.startOrderGroup(); PartitionIterator iter = UnfilteredPartitionIterators.filter(searchers.get(0).search(rc, orderGroup), FBUtilities.nowInSeconds())) + { + assertEquals(count, Util.size(iter)); + } + } +} diff --git a/test/unit/org/apache/cassandra/db/SerializationsTest.java b/test/unit/org/apache/cassandra/db/SerializationsTest.java deleted file mode 100644 index a280448752..0000000000 --- a/test/unit/org/apache/cassandra/db/SerializationsTest.java +++ /dev/null @@ -1,405 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - */ -package org.apache.cassandra.db; - -import org.apache.cassandra.AbstractSerializationsTester; -import org.apache.cassandra.SchemaLoader; -import org.apache.cassandra.Util; -import org.apache.cassandra.config.KSMetaData; -import org.apache.cassandra.db.composites.*; -import org.apache.cassandra.db.filter.*; -import org.apache.cassandra.db.marshal.AbstractType; -import org.apache.cassandra.db.marshal.BytesType; -import org.apache.cassandra.db.marshal.LongType; -import org.apache.cassandra.dht.AbstractBounds; -import org.apache.cassandra.dht.IPartitioner; -import org.apache.cassandra.dht.Range; -import org.apache.cassandra.dht.Token; -import org.apache.cassandra.exceptions.ConfigurationException; -import org.apache.cassandra.io.util.DataOutputStreamPlus; -import org.apache.cassandra.locator.SimpleStrategy; -import org.apache.cassandra.net.CallbackInfo; -import org.apache.cassandra.net.MessageIn; -import org.apache.cassandra.net.MessageOut; -import org.apache.cassandra.net.MessagingService; -import org.apache.cassandra.service.StorageService; -import org.apache.cassandra.utils.ByteBufferUtil; -import org.junit.BeforeClass; -import org.junit.Test; - -import java.io.DataInputStream; -import java.io.IOException; -import java.nio.ByteBuffer; -import java.util.*; - -public class SerializationsTest extends AbstractSerializationsTester -{ - Statics statics = new Statics(); - - private static final String KEYSPACE1 = "Keyspace1"; - private ByteBuffer startCol = ByteBufferUtil.bytes("Start"); - private ByteBuffer stopCol = ByteBufferUtil.bytes("Stop"); - private Composite emptyCol = Composites.EMPTY; - public NamesQueryFilter namesPred = new NamesQueryFilter(statics.NamedCols); - public NamesQueryFilter namesSCPred = new NamesQueryFilter(statics.NamedSCCols); - public SliceQueryFilter emptyRangePred = new SliceQueryFilter(emptyCol, emptyCol, false, 100); - public SliceQueryFilter nonEmptyRangePred = new SliceQueryFilter(CellNames.simpleDense(startCol), CellNames.simpleDense(stopCol), true, 100); - public SliceQueryFilter nonEmptyRangeSCPred = new SliceQueryFilter(CellNames.compositeDense(statics.SC, startCol), CellNames.compositeDense(statics.SC, stopCol), true, 100); - - @BeforeClass - public static void defineSchema() throws ConfigurationException - { - SchemaLoader.prepareServer(); - SchemaLoader.createKeyspace(KEYSPACE1, - SimpleStrategy.class, - KSMetaData.optsWithRF(1), - SchemaLoader.standardCFMD(KEYSPACE1, "Standard1"), - SchemaLoader.superCFMD(KEYSPACE1, "Super1", LongType.instance)); - } - - private void testRangeSliceCommandWrite() throws IOException - { - IPartitioner part = StorageService.getPartitioner(); - AbstractBounds bounds = Range.makeRowRange(part.getRandomToken(), part.getRandomToken()); - - RangeSliceCommand namesCmd = new RangeSliceCommand(statics.KS, "Standard1", statics.readTs, namesPred, bounds, 100); - MessageOut namesCmdMsg = namesCmd.createMessage(); - RangeSliceCommand emptyRangeCmd = new RangeSliceCommand(statics.KS, "Standard1", statics.readTs, emptyRangePred, bounds, 100); - MessageOut emptyRangeCmdMsg = emptyRangeCmd.createMessage(); - RangeSliceCommand regRangeCmd = new RangeSliceCommand(statics.KS, "Standard1", statics.readTs, nonEmptyRangePred, bounds, 100); - MessageOut regRangeCmdMsg = regRangeCmd.createMessage(); - RangeSliceCommand namesCmdSup = new RangeSliceCommand(statics.KS, "Super1", statics.readTs, namesSCPred, bounds, 100); - MessageOut namesCmdSupMsg = namesCmdSup.createMessage(); - RangeSliceCommand emptyRangeCmdSup = new RangeSliceCommand(statics.KS, "Super1", statics.readTs, emptyRangePred, bounds, 100); - MessageOut emptyRangeCmdSupMsg = emptyRangeCmdSup.createMessage(); - RangeSliceCommand regRangeCmdSup = new RangeSliceCommand(statics.KS, "Super1", statics.readTs, nonEmptyRangeSCPred, bounds, 100); - MessageOut regRangeCmdSupMsg = regRangeCmdSup.createMessage(); - - DataOutputStreamPlus out = getOutput("db.RangeSliceCommand.bin"); - namesCmdMsg.serialize(out, getVersion()); - emptyRangeCmdMsg.serialize(out, getVersion()); - regRangeCmdMsg.serialize(out, getVersion()); - namesCmdSupMsg.serialize(out, getVersion()); - emptyRangeCmdSupMsg.serialize(out, getVersion()); - regRangeCmdSupMsg.serialize(out, getVersion()); - out.close(); - - // test serializedSize - testSerializedSize(namesCmd, RangeSliceCommand.serializer); - testSerializedSize(emptyRangeCmd, RangeSliceCommand.serializer); - testSerializedSize(regRangeCmd, RangeSliceCommand.serializer); - testSerializedSize(namesCmdSup, RangeSliceCommand.serializer); - testSerializedSize(emptyRangeCmdSup, RangeSliceCommand.serializer); - testSerializedSize(regRangeCmdSup, RangeSliceCommand.serializer); - } - - @Test - public void testRangeSliceCommandRead() throws IOException - { - if (EXECUTE_WRITES) - testRangeSliceCommandWrite(); - - DataInputStream in = getInput("db.RangeSliceCommand.bin"); - for (int i = 0; i < 6; i++) - MessageIn.read(in, getVersion(), -1); - in.close(); - } - - private void testSliceByNamesReadCommandWrite() throws IOException - { - SliceByNamesReadCommand standardCmd = new SliceByNamesReadCommand(statics.KS, statics.Key, statics.StandardCF, statics.readTs, namesPred); - SliceByNamesReadCommand superCmd = new SliceByNamesReadCommand(statics.KS, statics.Key, statics.SuperCF, statics.readTs, namesSCPred); - - DataOutputStreamPlus out = getOutput("db.SliceByNamesReadCommand.bin"); - SliceByNamesReadCommand.serializer.serialize(standardCmd, out, getVersion()); - SliceByNamesReadCommand.serializer.serialize(superCmd, out, getVersion()); - ReadCommand.serializer.serialize(standardCmd, out, getVersion()); - ReadCommand.serializer.serialize(superCmd, out, getVersion()); - standardCmd.createMessage().serialize(out, getVersion()); - superCmd.createMessage().serialize(out, getVersion()); - out.close(); - - // test serializedSize - testSerializedSize(standardCmd, SliceByNamesReadCommand.serializer); - testSerializedSize(superCmd, SliceByNamesReadCommand.serializer); - } - - @Test - public void testSliceByNamesReadCommandRead() throws IOException - { - if (EXECUTE_WRITES) - testSliceByNamesReadCommandWrite(); - - DataInputStream in = getInput("db.SliceByNamesReadCommand.bin"); - assert SliceByNamesReadCommand.serializer.deserialize(in, getVersion()) != null; - assert SliceByNamesReadCommand.serializer.deserialize(in, getVersion()) != null; - assert ReadCommand.serializer.deserialize(in, getVersion()) != null; - assert ReadCommand.serializer.deserialize(in, getVersion()) != null; - assert MessageIn.read(in, getVersion(), -1) != null; - assert MessageIn.read(in, getVersion(), -1) != null; - in.close(); - } - - private void testSliceFromReadCommandWrite() throws IOException - { - SliceFromReadCommand standardCmd = new SliceFromReadCommand(statics.KS, statics.Key, statics.StandardCF, statics.readTs, nonEmptyRangePred); - SliceFromReadCommand superCmd = new SliceFromReadCommand(statics.KS, statics.Key, statics.SuperCF, statics.readTs, nonEmptyRangeSCPred); - - DataOutputStreamPlus out = getOutput("db.SliceFromReadCommand.bin"); - SliceFromReadCommand.serializer.serialize(standardCmd, out, getVersion()); - SliceFromReadCommand.serializer.serialize(superCmd, out, getVersion()); - ReadCommand.serializer.serialize(standardCmd, out, getVersion()); - ReadCommand.serializer.serialize(superCmd, out, getVersion()); - standardCmd.createMessage().serialize(out, getVersion()); - superCmd.createMessage().serialize(out, getVersion()); - - out.close(); - - // test serializedSize - testSerializedSize(standardCmd, SliceFromReadCommand.serializer); - testSerializedSize(superCmd, SliceFromReadCommand.serializer); - } - - @Test - public void testSliceFromReadCommandRead() throws IOException - { - if (EXECUTE_WRITES) - testSliceFromReadCommandWrite(); - - DataInputStream in = getInput("db.SliceFromReadCommand.bin"); - assert SliceFromReadCommand.serializer.deserialize(in, getVersion()) != null; - assert SliceFromReadCommand.serializer.deserialize(in, getVersion()) != null; - assert ReadCommand.serializer.deserialize(in, getVersion()) != null; - assert ReadCommand.serializer.deserialize(in, getVersion()) != null; - assert MessageIn.read(in, getVersion(), -1) != null; - assert MessageIn.read(in, getVersion(), -1) != null; - in.close(); - } - - private void testRowWrite() throws IOException - { - DataOutputStreamPlus out = getOutput("db.Row.bin"); - Row.serializer.serialize(statics.StandardRow, out, getVersion()); - Row.serializer.serialize(statics.SuperRow, out, getVersion()); - Row.serializer.serialize(statics.NullRow, out, getVersion()); - out.close(); - - // test serializedSize - testSerializedSize(statics.StandardRow, Row.serializer); - testSerializedSize(statics.SuperRow, Row.serializer); - testSerializedSize(statics.NullRow, Row.serializer); - } - - @Test - public void testRowRead() throws IOException - { - // Since every table creation generates different CF ID, - // we need to generate file every time - testRowWrite(); - - DataInputStream in = getInput("db.Row.bin"); - assert Row.serializer.deserialize(in, getVersion()) != null; - assert Row.serializer.deserialize(in, getVersion()) != null; - assert Row.serializer.deserialize(in, getVersion()) != null; - in.close(); - } - - private void testMutationWrite() throws IOException - { - Mutation standardRowRm = new Mutation(statics.KS, statics.StandardRow); - Mutation superRowRm = new Mutation(statics.KS, statics.SuperRow); - Mutation standardRm = new Mutation(statics.KS, statics.Key, statics.StandardCf); - Mutation superRm = new Mutation(statics.KS, statics.Key, statics.SuperCf); - Map mods = new HashMap(); - mods.put(statics.StandardCf.metadata().cfId, statics.StandardCf); - mods.put(statics.SuperCf.metadata().cfId, statics.SuperCf); - Mutation mixedRm = new Mutation(statics.KS, statics.Key, mods); - - DataOutputStreamPlus out = getOutput("db.RowMutation.bin"); - Mutation.serializer.serialize(standardRowRm, out, getVersion()); - Mutation.serializer.serialize(superRowRm, out, getVersion()); - Mutation.serializer.serialize(standardRm, out, getVersion()); - Mutation.serializer.serialize(superRm, out, getVersion()); - Mutation.serializer.serialize(mixedRm, out, getVersion()); - - standardRowRm.createMessage().serialize(out, getVersion()); - superRowRm.createMessage().serialize(out, getVersion()); - standardRm.createMessage().serialize(out, getVersion()); - superRm.createMessage().serialize(out, getVersion()); - mixedRm.createMessage().serialize(out, getVersion()); - - out.close(); - - // test serializedSize - testSerializedSize(standardRowRm, Mutation.serializer); - testSerializedSize(superRowRm, Mutation.serializer); - testSerializedSize(standardRm, Mutation.serializer); - testSerializedSize(superRm, Mutation.serializer); - testSerializedSize(mixedRm, Mutation.serializer); - } - - @Test - public void testMutationRead() throws IOException - { - // mutation deserialization requires being able to look up the keyspace in the schema, - // so we need to rewrite this each time. plus, CF ID is different for every run. - testMutationWrite(); - - DataInputStream in = getInput("db.RowMutation.bin"); - assert Mutation.serializer.deserialize(in, getVersion()) != null; - assert Mutation.serializer.deserialize(in, getVersion()) != null; - assert Mutation.serializer.deserialize(in, getVersion()) != null; - assert Mutation.serializer.deserialize(in, getVersion()) != null; - assert Mutation.serializer.deserialize(in, getVersion()) != null; - assert MessageIn.read(in, getVersion(), -1) != null; - assert MessageIn.read(in, getVersion(), -1) != null; - assert MessageIn.read(in, getVersion(), -1) != null; - assert MessageIn.read(in, getVersion(), -1) != null; - assert MessageIn.read(in, getVersion(), -1) != null; - in.close(); - } - - private void testTruncateWrite() throws IOException - { - Truncation tr = new Truncation(statics.KS, "Doesn't Really Matter"); - TruncateResponse aff = new TruncateResponse(statics.KS, "Doesn't Matter Either", true); - TruncateResponse neg = new TruncateResponse(statics.KS, "Still Doesn't Matter", false); - DataOutputStreamPlus out = getOutput("db.Truncation.bin"); - Truncation.serializer.serialize(tr, out, getVersion()); - TruncateResponse.serializer.serialize(aff, out, getVersion()); - TruncateResponse.serializer.serialize(neg, out, getVersion()); - - tr.createMessage().serialize(out, getVersion()); - aff.createMessage().serialize(out, getVersion()); - neg.createMessage().serialize(out, getVersion()); - // todo: notice how CF names weren't validated. - out.close(); - - // test serializedSize - testSerializedSize(tr, Truncation.serializer); - testSerializedSize(aff, TruncateResponse.serializer); - testSerializedSize(neg, TruncateResponse.serializer); - } - - @Test - public void testTruncateRead() throws IOException - { - if (EXECUTE_WRITES) - testTruncateWrite(); - - DataInputStream in = getInput("db.Truncation.bin"); - assert Truncation.serializer.deserialize(in, getVersion()) != null; - assert TruncateResponse.serializer.deserialize(in, getVersion()) != null; - assert TruncateResponse.serializer.deserialize(in, getVersion()) != null; - assert MessageIn.read(in, getVersion(), -1) != null; - - // set up some fake callbacks so deserialization knows that what it's deserializing is a TruncateResponse - MessagingService.instance().setCallbackForTests(1, new CallbackInfo(null, null, TruncateResponse.serializer, false)); - MessagingService.instance().setCallbackForTests(2, new CallbackInfo(null, null, TruncateResponse.serializer, false)); - - assert MessageIn.read(in, getVersion(), 1) != null; - assert MessageIn.read(in, getVersion(), 2) != null; - in.close(); - } - - private void testWriteResponseWrite() throws IOException - { - WriteResponse aff = new WriteResponse(); - WriteResponse neg = new WriteResponse(); - DataOutputStreamPlus out = getOutput("db.WriteResponse.bin"); - WriteResponse.serializer.serialize(aff, out, getVersion()); - WriteResponse.serializer.serialize(neg, out, getVersion()); - out.close(); - - // test serializedSize - testSerializedSize(aff, WriteResponse.serializer); - testSerializedSize(neg, WriteResponse.serializer); - } - - @Test - public void testWriteResponseRead() throws IOException - { - if (EXECUTE_WRITES) - testWriteResponseWrite(); - - DataInputStream in = getInput("db.WriteResponse.bin"); - assert WriteResponse.serializer.deserialize(in, getVersion()) != null; - assert WriteResponse.serializer.deserialize(in, getVersion()) != null; - in.close(); - } - - private static ByteBuffer bb(String s) - { - return ByteBufferUtil.bytes(s); - } - - private static CellName cn(String s) - { - return CellNames.simpleDense(ByteBufferUtil.bytes(s)); - } - - private static class Statics - { - private final String KS = KEYSPACE1; - private final ByteBuffer Key = ByteBufferUtil.bytes("Key01"); - private final SortedSet NamedCols = new TreeSet(new SimpleDenseCellNameType(BytesType.instance)) - {{ - add(CellNames.simpleDense(ByteBufferUtil.bytes("AAA"))); - add(CellNames.simpleDense(ByteBufferUtil.bytes("BBB"))); - add(CellNames.simpleDense(ByteBufferUtil.bytes("CCC"))); - }}; - private final ByteBuffer SC = ByteBufferUtil.bytes("SCName"); - private final SortedSet NamedSCCols = new TreeSet(new CompoundDenseCellNameType(Arrays.>asList(BytesType.instance, BytesType.instance))) - {{ - add(CellNames.compositeDense(SC, ByteBufferUtil.bytes("AAA"))); - add(CellNames.compositeDense(SC, ByteBufferUtil.bytes("BBB"))); - add(CellNames.compositeDense(SC, ByteBufferUtil.bytes("CCC"))); - }}; - private final String StandardCF = "Standard1"; - private final String SuperCF = "Super1"; - - private final long readTs = 1369935512292L; - - private final ColumnFamily StandardCf = ArrayBackedSortedColumns.factory.create(KS, StandardCF); - private final ColumnFamily SuperCf = ArrayBackedSortedColumns.factory.create(KS, SuperCF); - - private final Row StandardRow = new Row(Util.dk("key0"), StandardCf); - private final Row SuperRow = new Row(Util.dk("key1"), SuperCf); - private final Row NullRow = new Row(Util.dk("key2"), null); - - private Statics() - { - StandardCf.addColumn(new BufferCell(cn("aaaa"))); - StandardCf.addColumn(new BufferCell(cn("bbbb"), bb("bbbbb-value"))); - StandardCf.addColumn(new BufferCell(cn("cccc"), bb("ccccc-value"), 1000L)); - StandardCf.addColumn(new BufferDeletedCell(cn("dddd"), 500, 1000)); - StandardCf.addColumn(new BufferDeletedCell(cn("eeee"), bb("eeee-value"), 1001)); - StandardCf.addColumn(new BufferExpiringCell(cn("ffff"), bb("ffff-value"), 2000, 1000)); - StandardCf.addColumn(new BufferExpiringCell(cn("gggg"), bb("gggg-value"), 2001, 1000, 2002)); - - SuperCf.addColumn(new BufferCell(CellNames.compositeDense(SC, bb("aaaa")))); - SuperCf.addColumn(new BufferCell(CellNames.compositeDense(SC, bb("bbbb")), bb("bbbbb-value"))); - SuperCf.addColumn(new BufferCell(CellNames.compositeDense(SC, bb("cccc")), bb("ccccc-value"), 1000L)); - SuperCf.addColumn(new BufferDeletedCell(CellNames.compositeDense(SC, bb("dddd")), 500, 1000)); - SuperCf.addColumn(new BufferDeletedCell(CellNames.compositeDense(SC, bb("eeee")), bb("eeee-value"), 1001)); - SuperCf.addColumn(new BufferExpiringCell(CellNames.compositeDense(SC, bb("ffff")), bb("ffff-value"), 2000, 1000)); - SuperCf.addColumn(new BufferExpiringCell(CellNames.compositeDense(SC, bb("gggg")), bb("gggg-value"), 2001, 1000, 2002)); - } - } -} diff --git a/test/unit/org/apache/cassandra/db/TimeSortTest.java b/test/unit/org/apache/cassandra/db/TimeSortTest.java index 1d9fb10a14..8ae05ea957 100644 --- a/test/unit/org/apache/cassandra/db/TimeSortTest.java +++ b/test/unit/org/apache/cassandra/db/TimeSortTest.java @@ -18,134 +18,79 @@ */ package org.apache.cassandra.db; -import java.io.IOException; -import java.util.*; - -import org.junit.BeforeClass; +import java.util.Iterator; import org.junit.Test; -import static org.junit.Assert.assertEquals; -import org.apache.cassandra.SchemaLoader; -import static org.apache.cassandra.Util.cellname; -import static org.apache.cassandra.Util.getBytes; +import org.apache.cassandra.cql3.CQLTester; +import org.apache.cassandra.cql3.UntypedResultSet; import org.apache.cassandra.Util; -import org.apache.cassandra.config.KSMetaData; -import org.apache.cassandra.db.composites.*; -import org.apache.cassandra.db.filter.QueryFilter; -import org.apache.cassandra.exceptions.ConfigurationException; -import org.apache.cassandra.locator.SimpleStrategy; -import org.apache.cassandra.utils.ByteBufferUtil; +import static org.junit.Assert.assertEquals; - -public class TimeSortTest +public class TimeSortTest extends CQLTester { - private static final String KEYSPACE1 = "TimeSortTest"; - private static final String CF_STANDARD1 = "StandardLong1"; - - @BeforeClass - public static void defineSchema() throws ConfigurationException + @Test + public void testMixedSources() throws Throwable { - SchemaLoader.prepareServer(); - SchemaLoader.createKeyspace(KEYSPACE1, - SimpleStrategy.class, - KSMetaData.optsWithRF(1), - SchemaLoader.standardCFMD(KEYSPACE1, CF_STANDARD1)); + String tableName = createTable("CREATE TABLE %s (a int, b int, c int, PRIMARY KEY (a, b))"); + ColumnFamilyStore cfs = Keyspace.open(KEYSPACE).getColumnFamilyStore(tableName); + + execute("INSERT INTO %s (a, b, c) VALUES (?, ?, ?) USING TIMESTAMP ?", 0, 100, 0, 100L); + cfs.forceBlockingFlush(); + execute("INSERT INTO %s (a, b, c) VALUES (?, ?, ?) USING TIMESTAMP ?", 0, 0, 1, 0L); + + assertRows(execute("SELECT * FROM %s WHERE a = ? AND b >= ? LIMIT 1000", 0, 10), row(0, 100, 0)); } @Test - public void testMixedSources() + public void testTimeSort() throws Throwable { - Keyspace keyspace = Keyspace.open(KEYSPACE1); - ColumnFamilyStore cfStore = keyspace.getColumnFamilyStore(CF_STANDARD1); - Mutation rm; - DecoratedKey key = Util.dk("key0"); - - rm = new Mutation(KEYSPACE1, key.getKey()); - rm.add(CF_STANDARD1, cellname(100), ByteBufferUtil.bytes("a"), 100); - rm.applyUnsafe(); - cfStore.forceBlockingFlush(); - - rm = new Mutation(KEYSPACE1, key.getKey()); - rm.add(CF_STANDARD1, cellname(0), ByteBufferUtil.bytes("b"), 0); - rm.applyUnsafe(); - - ColumnFamily cf = cfStore.getColumnFamily(key, cellname(10), Composites.EMPTY, false, 1000, System.currentTimeMillis()); - Collection cells = cf.getSortedColumns(); - assert cells.size() == 1; - } - - @Test - public void testTimeSort() throws IOException - { - Keyspace keyspace = Keyspace.open(KEYSPACE1); - ColumnFamilyStore cfStore = keyspace.getColumnFamilyStore(CF_STANDARD1); + String tableName = createTable("CREATE TABLE %s (a int, b int, c int, PRIMARY KEY (a, b))"); + ColumnFamilyStore cfs = Keyspace.open(KEYSPACE).getColumnFamilyStore(tableName); for (int i = 900; i < 1000; ++i) - { - Mutation rm = new Mutation(KEYSPACE1, ByteBufferUtil.bytes(Integer.toString(i))); for (int j = 0; j < 8; ++j) - { - rm.add(CF_STANDARD1, cellname(j * 2), ByteBufferUtil.bytes("a"), j * 2); - } - rm.applyUnsafe(); - } + execute("INSERT INTO %s (a, b, c) VALUES (?, ?, ?) USING TIMESTAMP ?", i, j * 2, 0, (long)j * 2); - validateTimeSort(keyspace); - - cfStore.forceBlockingFlush(); - validateTimeSort(keyspace); + validateTimeSort(); + cfs.forceBlockingFlush(); + validateTimeSort(); // interleave some new data to test memtable + sstable DecoratedKey key = Util.dk("900"); - Mutation rm = new Mutation(KEYSPACE1, key.getKey()); for (int j = 0; j < 4; ++j) - { - rm.add(CF_STANDARD1, cellname(j * 2 + 1), ByteBufferUtil.bytes("b"), j * 2 + 1); - } - rm.applyUnsafe(); + execute("INSERT INTO %s (a, b, c) VALUES (?, ?, ?) USING TIMESTAMP ?", 900, j * 2 + 1, 1, (long)j * 2 + 1); + // and some overwrites - rm = new Mutation(KEYSPACE1, key.getKey()); - rm.add(CF_STANDARD1, cellname(0), ByteBufferUtil.bytes("c"), 100); - rm.add(CF_STANDARD1, cellname(10), ByteBufferUtil.bytes("c"), 100); - rm.applyUnsafe(); + execute("INSERT INTO %s (a, b, c) VALUES (?, ?, ?) USING TIMESTAMP ?", 900, 0, 2, 100L); + execute("INSERT INTO %s (a, b, c) VALUES (?, ?, ?) USING TIMESTAMP ?", 900, 10, 2, 100L); // verify - ColumnFamily cf = cfStore.getColumnFamily(key, cellname(0), Composites.EMPTY, false, 1000, System.currentTimeMillis()); - Collection cells = cf.getSortedColumns(); - assertEquals(12, cells.size()); - Iterator iter = cells.iterator(); - Cell cell; + UntypedResultSet results = execute("SELECT * FROM %s WHERE a = ? AND b >= ? LIMIT 1000", 900, 0); + assertEquals(12, results.size()); + Iterator iter = results.iterator(); for (int j = 0; j < 8; j++) { - cell = iter.next(); - assert cell.name().toByteBuffer().equals(getBytes(j)); + UntypedResultSet.Row row = iter.next(); + assertEquals(j, row.getInt("b")); } - TreeSet columnNames = new TreeSet(cfStore.getComparator()); - columnNames.add(cellname(10)); - columnNames.add(cellname(0)); - cf = cfStore.getColumnFamily(QueryFilter.getNamesFilter(Util.dk("900"), CF_STANDARD1, columnNames, System.currentTimeMillis())); - assert "c".equals(ByteBufferUtil.string(cf.getColumn(cellname(0)).value())); - assert "c".equals(ByteBufferUtil.string(cf.getColumn(cellname(10)).value())); + + assertRows(execute("SELECT * FROM %s WHERE a = ? AND b IN (?, ?)", 900, 0, 10), + row(900, 0, 2), + row(900, 10, 2)); } - private void validateTimeSort(Keyspace keyspace) + private void validateTimeSort() throws Throwable { for (int i = 900; i < 1000; ++i) { - DecoratedKey key = Util.dk(Integer.toString(i)); for (int j = 0; j < 8; j += 3) { - ColumnFamilyStore cfs = keyspace.getColumnFamilyStore(CF_STANDARD1); - ColumnFamily cf = cfs.getColumnFamily(key, cellname(j * 2), Composites.EMPTY, false, 1000, System.currentTimeMillis()); - Collection cells = cf.getSortedColumns(); - assert cells.size() == 8 - j; + UntypedResultSet results = execute("SELECT writetime(c) AS wt FROM %s WHERE a = ? AND b >= ? LIMIT 1000", i, j * 2); + assertEquals(8 - j, results.size()); int k = j; - for (Cell c : cells) - { - assertEquals((k++) * 2, c.timestamp()); - - } + for (UntypedResultSet.Row row : results) + assertEquals((k++) * 2, row.getLong("wt")); } } } diff --git a/test/unit/org/apache/cassandra/db/VerifyTest.java b/test/unit/org/apache/cassandra/db/VerifyTest.java index 27e99abf44..272d40f296 100644 --- a/test/unit/org/apache/cassandra/db/VerifyTest.java +++ b/test/unit/org/apache/cassandra/db/VerifyTest.java @@ -24,14 +24,15 @@ import com.google.common.base.Charsets; import org.apache.cassandra.OrderedJUnit4ClassRunner; import org.apache.cassandra.SchemaLoader; import org.apache.cassandra.Util; +import org.apache.cassandra.UpdateBuilder; import org.apache.cassandra.config.CFMetaData; import org.apache.cassandra.config.KSMetaData; -import org.apache.cassandra.db.columniterator.IdentityQueryFilter; import org.apache.cassandra.db.compaction.CompactionManager; import org.apache.cassandra.db.compaction.Verifier; import org.apache.cassandra.db.marshal.BytesType; import org.apache.cassandra.db.marshal.CounterColumnType; import org.apache.cassandra.db.marshal.UUIDType; +import org.apache.cassandra.db.partitions.PartitionIterator; import org.apache.cassandra.exceptions.ConfigurationException; import org.apache.cassandra.exceptions.WriteTimeoutException; import org.apache.cassandra.io.FSWriteError; @@ -54,8 +55,6 @@ import java.util.List; import java.util.zip.Adler32; import java.util.zip.CheckedInputStream; -import static org.apache.cassandra.Util.cellname; -import static org.apache.cassandra.Util.column; import static org.junit.Assert.fail; @RunWith(OrderedJUnit4ClassRunner.class) @@ -92,13 +91,13 @@ public class VerifyTest SchemaLoader.standardCFMD(KEYSPACE, CF4), SchemaLoader.standardCFMD(KEYSPACE, CORRUPT_CF), SchemaLoader.standardCFMD(KEYSPACE, CORRUPT_CF2), - SchemaLoader.standardCFMD(KEYSPACE, COUNTER_CF, BytesType.instance).defaultValidator(CounterColumnType.instance).compressionParameters(compressionParameters), - SchemaLoader.standardCFMD(KEYSPACE, COUNTER_CF2, BytesType.instance).defaultValidator(CounterColumnType.instance).compressionParameters(compressionParameters), - SchemaLoader.standardCFMD(KEYSPACE, COUNTER_CF3, BytesType.instance).defaultValidator(CounterColumnType.instance), - SchemaLoader.standardCFMD(KEYSPACE, COUNTER_CF4, BytesType.instance).defaultValidator(CounterColumnType.instance), - SchemaLoader.standardCFMD(KEYSPACE, CORRUPTCOUNTER_CF, BytesType.instance).defaultValidator(CounterColumnType.instance), - SchemaLoader.standardCFMD(KEYSPACE, CORRUPTCOUNTER_CF2, BytesType.instance).defaultValidator(CounterColumnType.instance), - SchemaLoader.standardCFMD(KEYSPACE, CF_UUID).keyValidator(UUIDType.instance)); + SchemaLoader.counterCFMD(KEYSPACE, COUNTER_CF).compressionParameters(compressionParameters), + SchemaLoader.counterCFMD(KEYSPACE, COUNTER_CF2).compressionParameters(compressionParameters), + SchemaLoader.counterCFMD(KEYSPACE, COUNTER_CF3), + SchemaLoader.counterCFMD(KEYSPACE, COUNTER_CF4), + SchemaLoader.counterCFMD(KEYSPACE, CORRUPTCOUNTER_CF), + SchemaLoader.counterCFMD(KEYSPACE, CORRUPTCOUNTER_CF2), + SchemaLoader.standardCFMD(KEYSPACE, CF_UUID, 0, UUIDType.instance)); } @@ -109,11 +108,11 @@ public class VerifyTest Keyspace keyspace = Keyspace.open(KEYSPACE); ColumnFamilyStore cfs = keyspace.getColumnFamilyStore(CF); - fillCF(cfs, KEYSPACE, CF, 2); + fillCF(cfs, 2); SSTableReader sstable = cfs.getSSTables().iterator().next(); - try(Verifier verifier = new Verifier(cfs, sstable, false)) + try (Verifier verifier = new Verifier(cfs, sstable, false)) { verifier.verify(false); } @@ -130,12 +129,11 @@ public class VerifyTest Keyspace keyspace = Keyspace.open(KEYSPACE); ColumnFamilyStore cfs = keyspace.getColumnFamilyStore(COUNTER_CF); - fillCounterCF(cfs, KEYSPACE, COUNTER_CF, 2); + fillCounterCF(cfs, 2); SSTableReader sstable = cfs.getSSTables().iterator().next(); - Verifier verifier = new Verifier(cfs, sstable, false); - try + try (Verifier verifier = new Verifier(cfs, sstable, false)) { verifier.verify(false); } @@ -152,12 +150,11 @@ public class VerifyTest Keyspace keyspace = Keyspace.open(KEYSPACE); ColumnFamilyStore cfs = keyspace.getColumnFamilyStore(CF2); - fillCF(cfs, KEYSPACE, CF2, 2); + fillCF(cfs, 2); SSTableReader sstable = cfs.getSSTables().iterator().next(); - Verifier verifier = new Verifier(cfs, sstable, false); - try + try (Verifier verifier = new Verifier(cfs, sstable, false)) { verifier.verify(true); } @@ -174,12 +171,11 @@ public class VerifyTest Keyspace keyspace = Keyspace.open(KEYSPACE); ColumnFamilyStore cfs = keyspace.getColumnFamilyStore(COUNTER_CF2); - fillCounterCF(cfs, KEYSPACE, COUNTER_CF2, 2); + fillCounterCF(cfs, 2); SSTableReader sstable = cfs.getSSTables().iterator().next(); - Verifier verifier = new Verifier(cfs, sstable, false); - try + try (Verifier verifier = new Verifier(cfs, sstable, false)) { verifier.verify(true); } @@ -196,12 +192,11 @@ public class VerifyTest Keyspace keyspace = Keyspace.open(KEYSPACE); ColumnFamilyStore cfs = keyspace.getColumnFamilyStore(CF3); - fillCF(cfs, KEYSPACE, CF3, 2); + fillCF(cfs, 2); SSTableReader sstable = cfs.getSSTables().iterator().next(); - Verifier verifier = new Verifier(cfs, sstable, false); - try + try (Verifier verifier = new Verifier(cfs, sstable, false)) { verifier.verify(false); } @@ -218,12 +213,11 @@ public class VerifyTest Keyspace keyspace = Keyspace.open(KEYSPACE); ColumnFamilyStore cfs = keyspace.getColumnFamilyStore(COUNTER_CF3); - fillCounterCF(cfs, KEYSPACE, COUNTER_CF3, 2); + fillCounterCF(cfs, 2); SSTableReader sstable = cfs.getSSTables().iterator().next(); - Verifier verifier = new Verifier(cfs, sstable, false); - try + try (Verifier verifier = new Verifier(cfs, sstable, false)) { verifier.verify(false); } @@ -240,12 +234,11 @@ public class VerifyTest Keyspace keyspace = Keyspace.open(KEYSPACE); ColumnFamilyStore cfs = keyspace.getColumnFamilyStore(CF4); - fillCF(cfs, KEYSPACE, CF4, 2); + fillCF(cfs, 2); SSTableReader sstable = cfs.getSSTables().iterator().next(); - Verifier verifier = new Verifier(cfs, sstable, false); - try + try (Verifier verifier = new Verifier(cfs, sstable, false)) { verifier.verify(true); } @@ -262,12 +255,11 @@ public class VerifyTest Keyspace keyspace = Keyspace.open(KEYSPACE); ColumnFamilyStore cfs = keyspace.getColumnFamilyStore(COUNTER_CF4); - fillCounterCF(cfs, KEYSPACE, COUNTER_CF4, 2); + fillCounterCF(cfs, 2); SSTableReader sstable = cfs.getSSTables().iterator().next(); - Verifier verifier = new Verifier(cfs, sstable, false); - try + try (Verifier verifier = new Verifier(cfs, sstable, false)) { verifier.verify(true); } @@ -285,9 +277,9 @@ public class VerifyTest Keyspace keyspace = Keyspace.open(KEYSPACE); ColumnFamilyStore cfs = keyspace.getColumnFamilyStore(CORRUPT_CF); - fillCF(cfs, KEYSPACE, CORRUPT_CF, 2); + fillCF(cfs, 2); - List rows = cfs.getRangeSlice(Util.range("", ""), null, new IdentityQueryFilter(), 1000); + Util.getAll(Util.cmd(cfs).build()); SSTableReader sstable = cfs.getSSTables().iterator().next(); @@ -298,8 +290,7 @@ public class VerifyTest writeChecksum(++correctChecksum, sstable.descriptor.filenameFor(Component.DIGEST)); - Verifier verifier = new Verifier(cfs, sstable, false); - try + try (Verifier verifier = new Verifier(cfs, sstable, false)) { verifier.verify(false); fail("Expected a CorruptSSTableException to be thrown"); @@ -315,15 +306,15 @@ public class VerifyTest Keyspace keyspace = Keyspace.open(KEYSPACE); ColumnFamilyStore cfs = keyspace.getColumnFamilyStore(CORRUPT_CF2); - fillCF(cfs, KEYSPACE, CORRUPT_CF2, 2); + fillCF(cfs, 2); - List rows = cfs.getRangeSlice(Util.range("", ""), null, new IdentityQueryFilter(), 1000); + Util.getAll(Util.cmd(cfs).build()); SSTableReader sstable = cfs.getSSTables().iterator().next(); // overwrite one row with garbage - long row0Start = sstable.getPosition(RowPosition.ForKey.get(ByteBufferUtil.bytes("0"), sstable.partitioner), SSTableReader.Operator.EQ).position; - long row1Start = sstable.getPosition(RowPosition.ForKey.get(ByteBufferUtil.bytes("1"), sstable.partitioner), SSTableReader.Operator.EQ).position; + long row0Start = sstable.getPosition(PartitionPosition.ForKey.get(ByteBufferUtil.bytes("0"), sstable.partitioner), SSTableReader.Operator.EQ).position; + long row1Start = sstable.getPosition(PartitionPosition.ForKey.get(ByteBufferUtil.bytes("1"), sstable.partitioner), SSTableReader.Operator.EQ).position; long startPosition = row0Start < row1Start ? row0Start : row1Start; long endPosition = row0Start < row1Start ? row1Start : row0Start; @@ -335,57 +326,53 @@ public class VerifyTest // Update the Digest to have the right Checksum writeChecksum(simpleFullChecksum(sstable.getFilename()), sstable.descriptor.filenameFor(Component.DIGEST)); - Verifier verifier = new Verifier(cfs, sstable, false); + try (Verifier verifier = new Verifier(cfs, sstable, false)) + { + // First a simple verify checking digest, which should succeed + try + { + verifier.verify(false); + } + catch (CorruptSSTableException err) + { + fail("Simple verify should have succeeded as digest matched"); + } - // First a simple verify checking digest, which should succeed - try - { - verifier.verify(false); - } - catch (CorruptSSTableException err) - { - fail("Simple verify should have succeeded as digest matched"); - } - - // Now try extended verify - try - { - verifier.verify(true); + // Now try extended verify + try + { + verifier.verify(true); + } + catch (CorruptSSTableException err) + { + return; + } + fail("Expected a CorruptSSTableException to be thrown"); } - catch (CorruptSSTableException err) - { - return; - } - fail("Expected a CorruptSSTableException to be thrown"); } - protected void fillCF(ColumnFamilyStore cfs, String keyspace, String columnFamily, int rowsPerSSTable) + protected void fillCF(ColumnFamilyStore cfs, int partitionsPerSSTable) { - for (int i = 0; i < rowsPerSSTable; i++) + for (int i = 0; i < partitionsPerSSTable; i++) { - String key = String.valueOf(i); - ColumnFamily cf = ArrayBackedSortedColumns.factory.create(keyspace, columnFamily); - cf.addColumn(column("c1", "1", 1L)); - cf.addColumn(column("c2", "2", 1L)); - Mutation rm = new Mutation(keyspace, ByteBufferUtil.bytes(key), cf); - rm.apply(); + UpdateBuilder.create(cfs.metadata, String.valueOf(i)) + .newRow("c1").add("val", "1") + .newRow("c2").add("val", "2") + .apply(); } cfs.forceBlockingFlush(); } - protected void fillCounterCF(ColumnFamilyStore cfs, String keyspace, String columnFamily, int rowsPerSSTable) throws WriteTimeoutException + protected void fillCounterCF(ColumnFamilyStore cfs, int partitionsPerSSTable) throws WriteTimeoutException { - for (int i = 0; i < rowsPerSSTable; i++) + for (int i = 0; i < partitionsPerSSTable; i++) { - String key = String.valueOf(i); - ColumnFamily cf = ArrayBackedSortedColumns.factory.create(keyspace, columnFamily); - Mutation rm = new Mutation(keyspace, ByteBufferUtil.bytes(key), cf); - rm.addCounter(columnFamily, cellname("Column1"), 100); - CounterMutation cm = new CounterMutation(rm, ConsistencyLevel.ONE); - cm.apply(); + UpdateBuilder.create(cfs.metadata, String.valueOf(i)) + .newRow("c1").add("val", 100L) + .apply(); } cfs.forceBlockingFlush(); diff --git a/test/unit/org/apache/cassandra/db/commitlog/CommitLogTestReplayer.java b/test/unit/org/apache/cassandra/db/commitlog/CommitLogTestReplayer.java index dc317c4a06..fed569f6ae 100644 --- a/test/unit/org/apache/cassandra/db/commitlog/CommitLogTestReplayer.java +++ b/test/unit/org/apache/cassandra/db/commitlog/CommitLogTestReplayer.java @@ -27,8 +27,8 @@ import com.google.common.base.Predicate; import org.junit.Assert; import org.apache.cassandra.config.DatabaseDescriptor; -import org.apache.cassandra.db.ColumnSerializer; import org.apache.cassandra.db.Mutation; +import org.apache.cassandra.db.rows.SerializationHelper; import org.apache.cassandra.io.util.FastByteArrayInputStream; /** @@ -67,7 +67,7 @@ public class CommitLogTestReplayer extends CommitLogReplayer { mutation = Mutation.serializer.deserialize(new DataInputStream(bufIn), desc.getMessagingVersion(), - ColumnSerializer.Flag.LOCAL); + SerializationHelper.Flag.LOCAL); Assert.assertTrue(processor.apply(mutation)); } catch (IOException e) diff --git a/test/unit/org/apache/cassandra/db/commitlog/CommitLogUpgradeTest.java b/test/unit/org/apache/cassandra/db/commitlog/CommitLogUpgradeTest.java index 1655078015..1289f43bc6 100644 --- a/test/unit/org/apache/cassandra/db/commitlog/CommitLogUpgradeTest.java +++ b/test/unit/org/apache/cassandra/db/commitlog/CommitLogUpgradeTest.java @@ -37,9 +37,11 @@ import org.junit.Test; import org.apache.cassandra.SchemaLoader; import org.apache.cassandra.config.CFMetaData; import org.apache.cassandra.config.Schema; -import org.apache.cassandra.db.Cell; -import org.apache.cassandra.db.ColumnFamily; import org.apache.cassandra.db.Mutation; +import org.apache.cassandra.db.rows.Cell; +import org.apache.cassandra.db.rows.Row; +import org.apache.cassandra.db.marshal.UTF8Type; +import org.apache.cassandra.db.partitions.PartitionUpdate; public class CommitLogUpgradeTest { @@ -51,7 +53,6 @@ public class CommitLogUpgradeTest static final String TABLE = "Standard1"; static final String KEYSPACE = "Keyspace1"; - static final String CELLNAME = "name"; @Test public void test20() throws Exception @@ -126,13 +127,13 @@ public class CommitLogUpgradeTest @Override public boolean apply(Mutation mutation) { - for (ColumnFamily cf : mutation.getColumnFamilies()) + for (PartitionUpdate update : mutation.getPartitionUpdates()) { - for (Cell c : cf.getSortedColumns()) + for (Row row : update) { - if (new String(c.name().toByteBuffer().array(), StandardCharsets.UTF_8).startsWith(CELLNAME)) + for (Cell cell : row) { - hash = hash(hash, c.value()); + hash = hash(hash, cell.value()); ++cells; } } diff --git a/test/unit/org/apache/cassandra/db/commitlog/CommitLogUpgradeTestMaker.java b/test/unit/org/apache/cassandra/db/commitlog/CommitLogUpgradeTestMaker.java index 7b07c8e3ee..cbae7e3586 100644 --- a/test/unit/org/apache/cassandra/db/commitlog/CommitLogUpgradeTestMaker.java +++ b/test/unit/org/apache/cassandra/db/commitlog/CommitLogUpgradeTestMaker.java @@ -36,6 +36,7 @@ import com.google.common.util.concurrent.RateLimiter; import org.apache.cassandra.SchemaLoader; import org.apache.cassandra.Util; +import org.apache.cassandra.UpdateBuilder; import org.apache.cassandra.config.DatabaseDescriptor; import org.apache.cassandra.config.Schema; import org.apache.cassandra.db.Mutation; @@ -231,18 +232,20 @@ public class CommitLogUpgradeTestMaker rl.acquire(); String ks = KEYSPACE; ByteBuffer key = randomBytes(16, tlr); - Mutation mutation = new Mutation(ks, key); + + UpdateBuilder builder = UpdateBuilder.create(Schema.instance.getCFMetaData(KEYSPACE, TABLE), Util.dk(key)); for (int ii = 0; ii < numCells; ii++) { int sz = randomSize ? tlr.nextInt(cellSize) : cellSize; ByteBuffer bytes = randomBytes(sz, tlr); - mutation.add(TABLE, Util.cellname(CELLNAME + ii), bytes, System.currentTimeMillis()); + builder.newRow("name" + ii).add("val", bytes); hash = hash(hash, bytes); ++cells; dataSize += sz; } - rp = commitLog.add(mutation); + + rp = commitLog.add((Mutation)builder.makeMutation()); counter.incrementAndGet(); } } diff --git a/test/unit/org/apache/cassandra/db/compaction/AntiCompactionTest.java b/test/unit/org/apache/cassandra/db/compaction/AntiCompactionTest.java index 235462b1c3..8a35494651 100644 --- a/test/unit/org/apache/cassandra/db/compaction/AntiCompactionTest.java +++ b/test/unit/org/apache/cassandra/db/compaction/AntiCompactionTest.java @@ -17,60 +17,64 @@ */ package org.apache.cassandra.db.compaction; -import org.apache.cassandra.db.lifecycle.LifecycleTransaction; -import org.apache.cassandra.utils.concurrent.Refs; -import static org.hamcrest.CoreMatchers.is; -import static org.junit.Assert.assertEquals; -import static org.junit.Assert.assertFalse; -import static org.junit.Assert.assertThat; -import static org.junit.Assert.assertTrue; - import java.io.File; import java.io.IOException; import java.util.Arrays; import java.util.Collection; import java.util.List; -import org.apache.cassandra.config.KSMetaData; -import org.apache.cassandra.exceptions.ConfigurationException; -import org.apache.cassandra.io.sstable.format.SSTableReader; -import org.apache.cassandra.io.sstable.format.SSTableWriter; -import org.apache.cassandra.locator.SimpleStrategy; +import com.google.common.collect.Iterables; +import com.google.common.util.concurrent.RateLimiter; +import junit.framework.Assert; import org.junit.BeforeClass; import org.junit.After; import org.junit.Test; -import org.apache.cassandra.SchemaLoader; -import org.apache.cassandra.Util; -import org.apache.cassandra.db.ArrayBackedSortedColumns; -import org.apache.cassandra.db.ColumnFamilyStore; -import org.apache.cassandra.db.DecoratedKey; -import org.apache.cassandra.db.Keyspace; -import org.apache.cassandra.db.Mutation; +import org.apache.cassandra.config.CFMetaData; +import org.apache.cassandra.config.KSMetaData; +import org.apache.cassandra.db.*; +import org.apache.cassandra.db.lifecycle.LifecycleTransaction; +import org.apache.cassandra.db.rows.RowStats; +import org.apache.cassandra.db.rows.UnfilteredRowIterator; import org.apache.cassandra.dht.ByteOrderedPartitioner.BytesToken; import org.apache.cassandra.dht.Range; import org.apache.cassandra.dht.Token; +import org.apache.cassandra.exceptions.ConfigurationException; import org.apache.cassandra.io.sstable.*; +import org.apache.cassandra.io.sstable.format.SSTableReader; +import org.apache.cassandra.io.sstable.format.SSTableWriter; +import org.apache.cassandra.locator.SimpleStrategy; import org.apache.cassandra.service.ActiveRepairService; -import org.apache.cassandra.service.StorageService; +import org.apache.cassandra.SchemaLoader; import org.apache.cassandra.utils.ByteBufferUtil; +import org.apache.cassandra.utils.concurrent.Refs; +import org.apache.cassandra.utils.FBUtilities; +import org.apache.cassandra.utils.concurrent.Refs; +import org.apache.cassandra.Util; +import org.apache.cassandra.UpdateBuilder; + +import static org.hamcrest.CoreMatchers.is; +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertFalse; +import static org.junit.Assert.assertThat; +import static org.junit.Assert.assertTrue; -import com.google.common.collect.Iterables; public class AntiCompactionTest { private static final String KEYSPACE1 = "AntiCompactionTest"; - private static final String CF = "Standard1"; - + private static final String CF = "AntiCompactionTest"; + private static CFMetaData cfm; @BeforeClass public static void defineSchema() throws ConfigurationException { SchemaLoader.prepareServer(); + cfm = SchemaLoader.standardCFMD(KEYSPACE1, CF); SchemaLoader.createKeyspace(KEYSPACE1, SimpleStrategy.class, KSMetaData.optsWithRF(1), - SchemaLoader.standardCFMD(KEYSPACE1, CF)); + cfm); } @After @@ -104,19 +108,19 @@ public class AntiCompactionTest assertEquals(2, store.getSSTables().size()); for (SSTableReader sstable : store.getSSTables()) { - try (ISSTableScanner scanner = sstable.getScanner()) + try (ISSTableScanner scanner = sstable.getScanner((RateLimiter) null)) { while (scanner.hasNext()) { - SSTableIdentityIterator row = (SSTableIdentityIterator) scanner.next(); + UnfilteredRowIterator row = scanner.next(); if (sstable.isRepaired()) { - assertTrue(range.contains(row.getKey().getToken())); + assertTrue(range.contains(row.partitionKey().getToken())); repairedKeys++; } else { - assertFalse(range.contains(row.getKey().getToken())); + assertFalse(range.contains(row.partitionKey().getToken())); nonRepairedKeys++; } } @@ -157,50 +161,50 @@ public class AntiCompactionTest private SSTableReader writeFile(ColumnFamilyStore cfs, int count) { - ArrayBackedSortedColumns cf = ArrayBackedSortedColumns.factory.create(cfs.metadata); - for (int i = 0; i < count; i++) - cf.addColumn(Util.column(String.valueOf(i), "a", 1)); File dir = cfs.directories.getDirectoryForNewSSTables(); String filename = cfs.getTempSSTablePath(dir); - try (SSTableWriter writer = SSTableWriter.create(filename, 0, 0);) + try (SSTableWriter writer = SSTableWriter.create(filename, 0, 0, new SerializationHeader(cfm, cfm.partitionColumns(), RowStats.NO_STATS))) { - for (int i = 0; i < count * 5; i++) - writer.append(StorageService.getPartitioner().decorateKey(ByteBufferUtil.bytes(i)), cf); + for (int i = 0; i < count; i++) + { + UpdateBuilder builder = UpdateBuilder.create(cfm, ByteBufferUtil.bytes(i)); + for (int j = 0; j < count * 5; j++) + builder.newRow("c" + j).add("val", "value1"); + writer.append(builder.build().unfilteredIterator()); + + } return writer.finish(true); } } public void generateSStable(ColumnFamilyStore store, String Suffix) { - long timestamp = System.currentTimeMillis(); - for (int i = 0; i < 10; i++) + for (int i = 0; i < 10; i++) { - DecoratedKey key = Util.dk(Integer.toString(i) + "-" + Suffix); - Mutation rm = new Mutation(KEYSPACE1, key.getKey()); - for (int j = 0; j < 10; j++) - rm.add("Standard1", Util.cellname(Integer.toString(j)), - ByteBufferUtil.EMPTY_BYTE_BUFFER, - timestamp, - 0); - rm.apply(); + String localSuffix = Integer.toString(i); + new RowUpdateBuilder(cfm, System.currentTimeMillis(), localSuffix + "-" + Suffix) + .clustering("c") + .add("val", "val" + localSuffix) + .build() + .applyUnsafe(); } store.forceBlockingFlush(); } @Test - public void antiCompactTenSTC() throws Exception + public void antiCompactTenSTC() throws InterruptedException, IOException { antiCompactTen("SizeTieredCompactionStrategy"); } @Test - public void antiCompactTenLC() throws Exception + public void antiCompactTenLC() throws InterruptedException, IOException { antiCompactTen("LeveledCompactionStrategy"); } - public void antiCompactTen(String compactionStrategy) throws Exception + public void antiCompactTen(String compactionStrategy) throws InterruptedException, IOException { Keyspace keyspace = Keyspace.open(KEYSPACE1); ColumnFamilyStore store = keyspace.getColumnFamilyStore(CF); @@ -232,22 +236,24 @@ public class AntiCompactionTest int nonRepairedKeys = 0; for (SSTableReader sstable : store.getSSTables()) { - try(ISSTableScanner scanner = sstable.getScanner()) + try (ISSTableScanner scanner = sstable.getScanner((RateLimiter) null)) { while (scanner.hasNext()) { - SSTableIdentityIterator row = (SSTableIdentityIterator) scanner.next(); - if (sstable.isRepaired()) + try (UnfilteredRowIterator row = scanner.next()) { - assertTrue(range.contains(row.getKey().getToken())); - assertEquals(repairedAt, sstable.getSSTableMetadata().repairedAt); - repairedKeys++; - } - else - { - assertFalse(range.contains(row.getKey().getToken())); - assertEquals(ActiveRepairService.UNREPAIRED_SSTABLE, sstable.getSSTableMetadata().repairedAt); - nonRepairedKeys++; + if (sstable.isRepaired()) + { + assertTrue(range.contains(row.partitionKey().getToken())); + assertEquals(repairedAt, sstable.getSSTableMetadata().repairedAt); + repairedKeys++; + } + else + { + assertFalse(range.contains(row.partitionKey().getToken())); + assertEquals(ActiveRepairService.UNREPAIRED_SSTABLE, sstable.getSSTableMetadata().repairedAt); + nonRepairedKeys++; + } } } } @@ -311,17 +317,13 @@ public class AntiCompactionTest Keyspace keyspace = Keyspace.open(KEYSPACE1); ColumnFamilyStore store = keyspace.getColumnFamilyStore(CF); store.disableAutoCompaction(); - long timestamp = System.currentTimeMillis(); for (int i = 0; i < 10; i++) { - DecoratedKey key = Util.dk(Integer.toString(i)); - Mutation rm = new Mutation(KEYSPACE1, key.getKey()); - for (int j = 0; j < 10; j++) - rm.add("Standard1", Util.cellname(Integer.toString(j)), - ByteBufferUtil.EMPTY_BYTE_BUFFER, - timestamp, - 0); - rm.apply(); + new RowUpdateBuilder(cfm, System.currentTimeMillis(), Integer.toString(i)) + .clustering("c") + .add("val", "val") + .build() + .applyUnsafe(); } store.forceBlockingFlush(); return store; diff --git a/test/unit/org/apache/cassandra/db/compaction/BlacklistingCompactionsTest.java b/test/unit/org/apache/cassandra/db/compaction/BlacklistingCompactionsTest.java index 2b6a62ae3f..5aa8e4f94c 100644 --- a/test/unit/org/apache/cassandra/db/compaction/BlacklistingCompactionsTest.java +++ b/test/unit/org/apache/cassandra/db/compaction/BlacklistingCompactionsTest.java @@ -24,17 +24,21 @@ package org.apache.cassandra.db.compaction; import java.io.RandomAccessFile; import java.util.*; -import org.apache.cassandra.io.sstable.format.SSTableReader; +import com.google.common.collect.Iterators; import org.junit.After; -import org.junit.AfterClass; import org.junit.BeforeClass; import org.junit.Test; +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertNotNull; + import org.apache.cassandra.SchemaLoader; import org.apache.cassandra.Util; import org.apache.cassandra.config.KSMetaData; import org.apache.cassandra.db.*; +import org.apache.cassandra.db.partitions.UnfilteredPartitionIterator; import org.apache.cassandra.exceptions.ConfigurationException; +import org.apache.cassandra.io.sstable.format.SSTableReader; import org.apache.cassandra.io.util.FileUtils; import org.apache.cassandra.locator.SimpleStrategy; import org.apache.cassandra.utils.ByteBufferUtil; @@ -42,7 +46,6 @@ import org.apache.cassandra.utils.ByteBufferUtil; import static org.junit.Assert.assertEquals; import static org.junit.Assert.assertNotNull; import static org.junit.Assert.assertTrue; -import static org.apache.cassandra.Util.cellname; public class BlacklistingCompactionsTest { @@ -107,22 +110,25 @@ public class BlacklistingCompactionsTest //test index corruption //now create a few new SSTables long maxTimestampExpected = Long.MIN_VALUE; - Set inserted = new HashSet(); + Set inserted = new HashSet<>(); + for (int j = 0; j < SSTABLES; j++) { for (int i = 0; i < ROWS_PER_SSTABLE; i++) { - DecoratedKey key = Util.dk(String.valueOf(i % 2)); - Mutation rm = new Mutation(KEYSPACE1, key.getKey()); + DecoratedKey key = Util.dk(String.valueOf(i)); long timestamp = j * ROWS_PER_SSTABLE + i; - rm.add("Standard1", cellname(i / 2), ByteBufferUtil.EMPTY_BYTE_BUFFER, timestamp); + new RowUpdateBuilder(cfs.metadata, timestamp, key.getKey()) + .clustering("cols" + "i") + .add("val", "val" + i) + .build() + .applyUnsafe(); maxTimestampExpected = Math.max(timestamp, maxTimestampExpected); - rm.applyUnsafe(); inserted.add(key); } cfs.forceBlockingFlush(); CompactionsTest.assertMaxTimestamp(cfs, maxTimestampExpected); - assertEquals(inserted.toString(), inserted.size(), Util.getRangeSlice(cfs).size()); + assertEquals(inserted.toString(), inserted.size(), Util.getAll(Util.cmd(cfs).build()).size()); } Collection sstables = cfs.getSSTables(); @@ -132,20 +138,21 @@ public class BlacklistingCompactionsTest // corrupt first 'sstablesToCorrupt' SSTables for (SSTableReader sstable : sstables) { - if(currentSSTable + 1 > sstablesToCorrupt) + if (currentSSTable + 1 > sstablesToCorrupt) break; RandomAccessFile raf = null; try { + int corruptionSize = 50; raf = new RandomAccessFile(sstable.getFilename(), "rw"); assertNotNull(raf); - assertTrue(raf.length() > 20); - raf.seek(new Random().nextInt((int)(raf.length() - 20))); + assertTrue(raf.length() > corruptionSize); + raf.seek(new Random().nextInt((int)(raf.length() - corruptionSize))); // We want to write something large enough that the corruption cannot get undetected // (even without compression) - byte[] corruption = new byte[20]; + byte[] corruption = new byte[corruptionSize]; Arrays.fill(corruption, (byte)0xFF); raf.write(corruption); @@ -175,8 +182,6 @@ public class BlacklistingCompactionsTest failures++; continue; } - - assertEquals(sstablesToCorrupt + 1, cfs.getSSTables().size()); break; } diff --git a/test/unit/org/apache/cassandra/db/compaction/CompactionAwareWriterTest.java b/test/unit/org/apache/cassandra/db/compaction/CompactionAwareWriterTest.java index 2b6f575755..0bcd603b1b 100644 --- a/test/unit/org/apache/cassandra/db/compaction/CompactionAwareWriterTest.java +++ b/test/unit/org/apache/cassandra/db/compaction/CompactionAwareWriterTest.java @@ -21,66 +21,61 @@ import java.nio.ByteBuffer; import java.util.*; import com.google.common.primitives.Longs; -import org.junit.Before; -import org.junit.BeforeClass; -import org.junit.Test; +import org.junit.*; -import org.apache.cassandra.SchemaLoader; -import org.apache.cassandra.Util; -import org.apache.cassandra.config.KSMetaData; -import org.apache.cassandra.db.Cell; -import org.apache.cassandra.db.ColumnFamily; -import org.apache.cassandra.db.ColumnFamilyStore; -import org.apache.cassandra.db.DecoratedKey; -import org.apache.cassandra.db.Keyspace; -import org.apache.cassandra.db.Mutation; +import org.apache.cassandra.cql3.CQLTester; +import org.apache.cassandra.cql3.QueryProcessor; +import org.apache.cassandra.db.*; import org.apache.cassandra.db.compaction.writers.CompactionAwareWriter; import org.apache.cassandra.db.compaction.writers.DefaultCompactionWriter; import org.apache.cassandra.db.compaction.writers.MajorLeveledCompactionWriter; import org.apache.cassandra.db.compaction.writers.MaxSSTableSizeWriter; import org.apache.cassandra.db.compaction.writers.SplittingSizeTieredCompactionWriter; -import org.apache.cassandra.db.filter.QueryFilter; import org.apache.cassandra.db.lifecycle.LifecycleTransaction; import org.apache.cassandra.exceptions.ConfigurationException; import org.apache.cassandra.io.sstable.ISSTableScanner; import org.apache.cassandra.io.sstable.format.SSTableReader; import org.apache.cassandra.locator.SimpleStrategy; +import org.apache.cassandra.utils.FBUtilities; +import org.apache.cassandra.utils.UUIDGen; import static org.junit.Assert.assertEquals; -public class CompactionAwareWriterTest +public class CompactionAwareWriterTest extends CQLTester { - private static String KEYSPACE1 = "CompactionAwareWriterTest"; - private static String CF = "Standard1"; + private static final String KEYSPACE = "cawt_keyspace"; + private static final String TABLE = "cawt_table"; + + private static final int ROW_PER_PARTITION = 10; @BeforeClass - public static void defineSchema() throws ConfigurationException + public static void beforeClass() throws Throwable { - SchemaLoader.prepareServer(); - SchemaLoader.createKeyspace(KEYSPACE1, - SimpleStrategy.class, - KSMetaData.optsWithRF(1), - SchemaLoader.standardCFMD(KEYSPACE1, CF)); - + // Disabling durable write since we don't care + schemaChange("CREATE KEYSPACE IF NOT EXISTS " + KEYSPACE + " WITH replication = {'class': 'SimpleStrategy', 'replication_factor': '1'} AND durable_writes=false"); + schemaChange(String.format("CREATE TABLE %s.%s (k int, t int, v blob, PRIMARY KEY (k, t))", KEYSPACE, TABLE)); } - @Before - public void clear() + @AfterClass + public static void tearDownClass() { - // avoid one test affecting the next one - Keyspace ks = Keyspace.open(KEYSPACE1); - ColumnFamilyStore cfs = ks.getColumnFamilyStore(CF); - cfs.clearUnsafe(); + QueryProcessor.executeInternal("DROP KEYSPACE IF EXISTS " + KEYSPACE); + } + + private ColumnFamilyStore getColumnFamilyStore() + { + return Keyspace.open(KEYSPACE).getColumnFamilyStore(TABLE); } @Test - public void testDefaultCompactionWriter() + public void testDefaultCompactionWriter() throws Throwable { - Keyspace ks = Keyspace.open(KEYSPACE1); - ColumnFamilyStore cfs = ks.getColumnFamilyStore(CF); + Keyspace ks = Keyspace.open(KEYSPACE); + ColumnFamilyStore cfs = ks.getColumnFamilyStore(TABLE); + int rowCount = 1000; cfs.disableAutoCompaction(); - populate(cfs, rowCount); + populate(rowCount); LifecycleTransaction txn = cfs.getTracker().tryModify(cfs.getSSTables(), OperationType.COMPACTION); long beforeSize = txn.originals().iterator().next().onDiskLength(); CompactionAwareWriter writer = new DefaultCompactionWriter(cfs, txn, txn.originals(), false, OperationType.COMPACTION); @@ -93,13 +88,12 @@ public class CompactionAwareWriterTest } @Test - public void testMaxSSTableSizeWriter() + public void testMaxSSTableSizeWriter() throws Throwable { - Keyspace ks = Keyspace.open(KEYSPACE1); - ColumnFamilyStore cfs = ks.getColumnFamilyStore(CF); + ColumnFamilyStore cfs = getColumnFamilyStore(); cfs.disableAutoCompaction(); int rowCount = 1000; - populate(cfs, rowCount); + populate(rowCount); LifecycleTransaction txn = cfs.getTracker().tryModify(cfs.getSSTables(), OperationType.COMPACTION); long beforeSize = txn.originals().iterator().next().onDiskLength(); int sstableSize = (int)beforeSize/10; @@ -110,14 +104,14 @@ public class CompactionAwareWriterTest validateData(cfs, rowCount); cfs.truncateBlocking(); } + @Test - public void testSplittingSizeTieredCompactionWriter() + public void testSplittingSizeTieredCompactionWriter() throws Throwable { - Keyspace ks = Keyspace.open(KEYSPACE1); - ColumnFamilyStore cfs = ks.getColumnFamilyStore(CF); + ColumnFamilyStore cfs = getColumnFamilyStore(); cfs.disableAutoCompaction(); int rowCount = 10000; - populate(cfs, rowCount); + populate(rowCount); LifecycleTransaction txn = cfs.getTracker().tryModify(cfs.getSSTables(), OperationType.COMPACTION); long beforeSize = txn.originals().iterator().next().onDiskLength(); CompactionAwareWriter writer = new SplittingSizeTieredCompactionWriter(cfs, txn, txn.originals(), OperationType.COMPACTION, 0); @@ -146,14 +140,13 @@ public class CompactionAwareWriterTest } @Test - public void testMajorLeveledCompactionWriter() + public void testMajorLeveledCompactionWriter() throws Throwable { - Keyspace ks = Keyspace.open(KEYSPACE1); - ColumnFamilyStore cfs = ks.getColumnFamilyStore(CF); + ColumnFamilyStore cfs = getColumnFamilyStore(); cfs.disableAutoCompaction(); int rowCount = 20000; int targetSSTableCount = 50; - populate(cfs, rowCount); + populate(rowCount); LifecycleTransaction txn = cfs.getTracker().tryModify(cfs.getSSTables(), OperationType.COMPACTION); long beforeSize = txn.originals().iterator().next().onDiskLength(); int sstableSize = (int)beforeSize/targetSSTableCount; @@ -179,14 +172,14 @@ public class CompactionAwareWriterTest { assert txn.originals().size() == 1; int rowsWritten = 0; - try (AbstractCompactionStrategy.ScannerList scanners = cfs.getCompactionStrategyManager().getScanners(txn.originals())) + int nowInSec = FBUtilities.nowInSeconds(); + try (AbstractCompactionStrategy.ScannerList scanners = cfs.getCompactionStrategyManager().getScanners(txn.originals()); + CompactionController controller = new CompactionController(cfs, txn.originals(), cfs.gcBefore(nowInSec)); + CompactionIterator ci = new CompactionIterator(OperationType.COMPACTION, scanners.scanners, controller, nowInSec, UUIDGen.getTimeUUID())) { - CompactionController controller = new CompactionController(cfs, txn.originals(), cfs.gcBefore(System.currentTimeMillis())); - ISSTableScanner scanner = scanners.scanners.get(0); - while(scanner.hasNext()) + while (ci.hasNext()) { - AbstractCompactedRow row = new LazilyCompactedRow(controller, Arrays.asList(scanner.next())); - if (writer.append(row)) + if (writer.append(ci.next())) rowsWritten++; } } @@ -194,22 +187,17 @@ public class CompactionAwareWriterTest return rowsWritten; } - private void populate(ColumnFamilyStore cfs, int count) + private void populate(int count) throws Throwable { - long timestamp = System.currentTimeMillis(); byte [] payload = new byte[1000]; new Random().nextBytes(payload); ByteBuffer b = ByteBuffer.wrap(payload); + for (int i = 0; i < count; i++) - { - DecoratedKey key = Util.dk(Integer.toString(i)); - Mutation rm = new Mutation(KEYSPACE1, key.getKey()); - for (int j = 0; j < 10; j++) - rm.add(CF, Util.cellname(Integer.toString(j)), - b, - timestamp); - rm.applyUnsafe(); - } + for (int j = 0; j < ROW_PER_PARTITION; j++) + execute(String.format("INSERT INTO %s.%s(k, t, v) VALUES (?, ?, ?)", KEYSPACE, TABLE), i, j, b); + + ColumnFamilyStore cfs = getColumnFamilyStore(); cfs.forceBlockingFlush(); if (cfs.getSSTables().size() > 1) { @@ -225,20 +213,16 @@ public class CompactionAwareWriterTest } assert cfs.getSSTables().size() == 1 : cfs.getSSTables(); } - private void validateData(ColumnFamilyStore cfs, int rowCount) + + private void validateData(ColumnFamilyStore cfs, int rowCount) throws Throwable { for (int i = 0; i < rowCount; i++) { - ColumnFamily cf = cfs.getTopLevelColumns(QueryFilter.getIdentityFilter(Util.dk(Integer.toString(i)), CF, System.currentTimeMillis()), Integer.MAX_VALUE); - Iterator iter = cf.iterator(); - int cellCount = 0; - while (iter.hasNext()) - { - Cell c = iter.next(); - assertEquals(Util.cellname(Integer.toString(cellCount)), c.name()); - cellCount++; - } - assertEquals(10, cellCount); + Object[][] expected = new Object[ROW_PER_PARTITION][]; + for (int j = 0; j < ROW_PER_PARTITION; j++) + expected[j] = row(i, j); + + assertRows(execute(String.format("SELECT k, t FROM %s.%s WHERE k = :i", KEYSPACE, TABLE), i), expected); } } } diff --git a/test/unit/org/apache/cassandra/db/compaction/CompactionsPurgeTest.java b/test/unit/org/apache/cassandra/db/compaction/CompactionsPurgeTest.java index d03c35a224..09ac021530 100644 --- a/test/unit/org/apache/cassandra/db/compaction/CompactionsPurgeTest.java +++ b/test/unit/org/apache/cassandra/db/compaction/CompactionsPurgeTest.java @@ -21,35 +21,29 @@ package org.apache.cassandra.db.compaction; import java.util.Collection; import java.util.concurrent.ExecutionException; -import org.apache.cassandra.cache.CachingOptions; -import org.apache.cassandra.config.CFMetaData; -import org.apache.cassandra.config.KSMetaData; -import org.apache.cassandra.db.*; - -import org.apache.cassandra.exceptions.ConfigurationException; -import org.apache.cassandra.io.sstable.format.SSTableReader; -import org.apache.cassandra.locator.SimpleStrategy; import org.junit.BeforeClass; import org.junit.Test; import org.apache.cassandra.SchemaLoader; -import org.apache.cassandra.cql3.UntypedResultSet; -import org.apache.cassandra.db.filter.QueryFilter; import org.apache.cassandra.Util; - -import static org.junit.Assert.assertEquals; -import static org.apache.cassandra.db.KeyspaceTest.assertColumns; -import static org.junit.Assert.assertFalse; -import static org.junit.Assert.assertTrue; -import static org.junit.Assert.assertNotNull; -import static org.junit.Assert.assertNull; - -import static org.apache.cassandra.cql3.QueryProcessor.executeInternal; - -import static org.apache.cassandra.Util.cellname; +import org.apache.cassandra.cache.CachingOptions; +import org.apache.cassandra.config.CFMetaData; +import org.apache.cassandra.config.KSMetaData; +import org.apache.cassandra.cql3.QueryProcessor; +import org.apache.cassandra.cql3.UntypedResultSet; +import org.apache.cassandra.db.*; +import org.apache.cassandra.db.rows.Row; +import org.apache.cassandra.db.partitions.ArrayBackedPartition; +import org.apache.cassandra.db.partitions.PartitionUpdate; +import org.apache.cassandra.exceptions.ConfigurationException; +import org.apache.cassandra.io.sstable.format.SSTableReader; +import org.apache.cassandra.locator.SimpleStrategy; import org.apache.cassandra.utils.ByteBufferUtil; import org.apache.cassandra.utils.FBUtilities; +import static org.apache.cassandra.Util.dk; +import static org.junit.Assert.*; + public class CompactionsPurgeTest { @@ -67,26 +61,26 @@ public class CompactionsPurgeTest { SchemaLoader.prepareServer(); SchemaLoader.createKeyspace(KEYSPACE1, - SimpleStrategy.class, - KSMetaData.optsWithRF(1), - SchemaLoader.standardCFMD(KEYSPACE1, CF_STANDARD1), - SchemaLoader.standardCFMD(KEYSPACE1, CF_STANDARD2)); + SimpleStrategy.class, + KSMetaData.optsWithRF(1), + SchemaLoader.standardCFMD(KEYSPACE1, CF_STANDARD1), + SchemaLoader.standardCFMD(KEYSPACE1, CF_STANDARD2)); SchemaLoader.createKeyspace(KEYSPACE2, - SimpleStrategy.class, - KSMetaData.optsWithRF(1), - SchemaLoader.standardCFMD(KEYSPACE2, CF_STANDARD1)); + SimpleStrategy.class, + KSMetaData.optsWithRF(1), + SchemaLoader.standardCFMD(KEYSPACE2, CF_STANDARD1)); SchemaLoader.createKeyspace(KEYSPACE_CACHED, - SimpleStrategy.class, - KSMetaData.optsWithRF(1), - SchemaLoader.standardCFMD(KEYSPACE_CACHED, CF_CACHED).caching(CachingOptions.ALL)); + SimpleStrategy.class, + KSMetaData.optsWithRF(1), + SchemaLoader.standardCFMD(KEYSPACE_CACHED, CF_CACHED).caching(CachingOptions.ALL)); SchemaLoader.createKeyspace(KEYSPACE_CQL, - SimpleStrategy.class, - KSMetaData.optsWithRF(1), - CFMetaData.compile("CREATE TABLE " + CF_CQL + " (" - + "k int PRIMARY KEY," - + "v1 text," - + "v2 int" - + ")", KEYSPACE_CQL)); + SimpleStrategy.class, + KSMetaData.optsWithRF(1), + CFMetaData.compile("CREATE TABLE " + CF_CQL + " (" + + "k int PRIMARY KEY," + + "v1 text," + + "v2 int" + + ")", KEYSPACE_CQL)); } @Test @@ -98,39 +92,40 @@ public class CompactionsPurgeTest String cfName = "Standard1"; ColumnFamilyStore cfs = keyspace.getColumnFamilyStore(cfName); - DecoratedKey key = Util.dk("key1"); - Mutation rm; + String key = "key1"; // inserts - rm = new Mutation(KEYSPACE1, key.getKey()); for (int i = 0; i < 10; i++) { - rm.add(cfName, cellname(String.valueOf(i)), ByteBufferUtil.EMPTY_BYTE_BUFFER, 0); + RowUpdateBuilder builder = new RowUpdateBuilder(cfs.metadata, 0, key); + builder.clustering(String.valueOf(i)) + .add("val", ByteBufferUtil.EMPTY_BYTE_BUFFER) + .build().applyUnsafe(); } - rm.applyUnsafe(); + cfs.forceBlockingFlush(); // deletes for (int i = 0; i < 10; i++) { - rm = new Mutation(KEYSPACE1, key.getKey()); - rm.delete(cfName, cellname(String.valueOf(i)), 1); - rm.applyUnsafe(); + RowUpdateBuilder.deleteRow(cfs.metadata, 1, key, String.valueOf(i)).applyUnsafe(); } cfs.forceBlockingFlush(); // resurrect one column - rm = new Mutation(KEYSPACE1, key.getKey()); - rm.add(cfName, cellname(String.valueOf(5)), ByteBufferUtil.EMPTY_BYTE_BUFFER, 2); - rm.applyUnsafe(); + RowUpdateBuilder builder = new RowUpdateBuilder(cfs.metadata, 2, key); + builder.clustering(String.valueOf(5)) + .add("val", ByteBufferUtil.EMPTY_BYTE_BUFFER) + .build().applyUnsafe(); + cfs.forceBlockingFlush(); // major compact and test that all columns but the resurrected one is completely gone FBUtilities.waitOnFutures(CompactionManager.instance.submitMaximal(cfs, Integer.MAX_VALUE, false)); - cfs.invalidateCachedRow(key); - ColumnFamily cf = cfs.getColumnFamily(QueryFilter.getIdentityFilter(key, cfName, System.currentTimeMillis())); - assertColumns(cf, "5"); - assertNotNull(cf.getColumn(cellname(String.valueOf(5)))); + cfs.invalidateCachedPartition(dk(key)); + + ArrayBackedPartition partition = Util.getOnlyPartitionUnfiltered(Util.cmd(cfs, key).build()); + assertEquals(1, partition.rowCount()); } @Test @@ -142,26 +137,25 @@ public class CompactionsPurgeTest String cfName = "Standard1"; ColumnFamilyStore cfs = keyspace.getColumnFamilyStore(cfName); - Mutation rm; for (int k = 1; k <= 2; ++k) { - DecoratedKey key = Util.dk("key" + k); + String key = "key" + k; // inserts - rm = new Mutation(KEYSPACE2, key.getKey()); for (int i = 0; i < 10; i++) { - rm.add(cfName, cellname(String.valueOf(i)), ByteBufferUtil.EMPTY_BYTE_BUFFER, 0); + RowUpdateBuilder builder = new RowUpdateBuilder(cfs.metadata, 0, key); + builder.clustering(String.valueOf(i)) + .add("val", ByteBufferUtil.EMPTY_BYTE_BUFFER) + .build().applyUnsafe(); } - rm.applyUnsafe(); cfs.forceBlockingFlush(); // deletes for (int i = 0; i < 10; i++) { - rm = new Mutation(KEYSPACE2, key.getKey()); - rm.delete(cfName, cellname(String.valueOf(i)), 1); - rm.applyUnsafe(); + RowUpdateBuilder.deleteRow(cfs.metadata, 1, key, String.valueOf(i)).applyUnsafe(); } + cfs.forceBlockingFlush(); } @@ -172,21 +166,23 @@ public class CompactionsPurgeTest // for first key. Then submit minor compaction on remembered sstables. cfs.forceBlockingFlush(); Collection sstablesIncomplete = cfs.getSSTables(); - rm = new Mutation(KEYSPACE2, key1.getKey()); - rm.add(cfName, cellname(String.valueOf(5)), ByteBufferUtil.EMPTY_BYTE_BUFFER, 2); - rm.applyUnsafe(); + + RowUpdateBuilder builder = new RowUpdateBuilder(cfs.metadata, 2, "key1"); + builder.clustering(String.valueOf(5)) + .add("val", ByteBufferUtil.EMPTY_BYTE_BUFFER) + .build().applyUnsafe(); + cfs.forceBlockingFlush(); cfs.getCompactionStrategyManager().getUserDefinedTask(sstablesIncomplete, Integer.MAX_VALUE).execute(null); // verify that minor compaction does GC when key is provably not // present in a non-compacted sstable - ColumnFamily cf = cfs.getColumnFamily(QueryFilter.getIdentityFilter(key2, cfName, System.currentTimeMillis())); - assertNull(cf); + Util.assertEmpty(Util.cmd(cfs, key2).build()); // verify that minor compaction still GC when key is present // in a non-compacted sstable but the timestamp ensures we won't miss anything - cf = cfs.getColumnFamily(QueryFilter.getIdentityFilter(key1, cfName, System.currentTimeMillis())); - assertEquals(1, cf.getColumnCount()); + ArrayBackedPartition partition = Util.getOnlyPartitionUnfiltered(Util.cmd(cfs, key1).build()); + assertEquals(1, partition.rowCount()); } /** @@ -200,26 +196,28 @@ public class CompactionsPurgeTest Keyspace keyspace = Keyspace.open(KEYSPACE2); String cfName = "Standard1"; ColumnFamilyStore cfs = keyspace.getColumnFamilyStore(cfName); - Mutation rm; - DecoratedKey key3 = Util.dk("key3"); + String key3 = "key3"; // inserts - rm = new Mutation(KEYSPACE2, key3.getKey()); - rm.add(cfName, cellname("c1"), ByteBufferUtil.EMPTY_BYTE_BUFFER, 8); - rm.add(cfName, cellname("c2"), ByteBufferUtil.EMPTY_BYTE_BUFFER, 8); - rm.applyUnsafe(); + new RowUpdateBuilder(cfs.metadata, 8, key3) + .clustering("c1") + .add("val", ByteBufferUtil.EMPTY_BYTE_BUFFER) + .build().applyUnsafe(); + + new RowUpdateBuilder(cfs.metadata, 8, key3) + .clustering("c2") + .add("val", ByteBufferUtil.EMPTY_BYTE_BUFFER) + .build().applyUnsafe(); + cfs.forceBlockingFlush(); // delete c1 - rm = new Mutation(KEYSPACE2, key3.getKey()); - rm.delete(cfName, cellname("c1"), 10); - rm.applyUnsafe(); + RowUpdateBuilder.deleteRow(cfs.metadata, 10, key3, "c1").applyUnsafe(); + cfs.forceBlockingFlush(); Collection sstablesIncomplete = cfs.getSSTables(); // delete c2 so we have new delete in a diffrent SSTable - rm = new Mutation(KEYSPACE2, key3.getKey()); - rm.delete(cfName, cellname("c2"), 9); - rm.applyUnsafe(); + RowUpdateBuilder.deleteRow(cfs.metadata, 9, key3, "c2").applyUnsafe(); cfs.forceBlockingFlush(); // compact the sstables with the c1/c2 data and the c1 tombstone @@ -227,9 +225,10 @@ public class CompactionsPurgeTest // We should have both the c1 and c2 tombstones still. Since the min timestamp in the c2 tombstone // sstable is older than the c1 tombstone, it is invalid to throw out the c1 tombstone. - ColumnFamily cf = cfs.getColumnFamily(QueryFilter.getIdentityFilter(key3, cfName, System.currentTimeMillis())); - assertFalse(cf.getColumn(cellname("c2")).isLive()); - assertEquals(2, cf.getColumnCount()); + ArrayBackedPartition partition = Util.getOnlyPartitionUnfiltered(Util.cmd(cfs, key3).build()); + assertEquals(2, partition.rowCount()); + for (Row row : partition) + assertFalse(row.hasLiveData(FBUtilities.nowInSeconds())); } @Test @@ -241,23 +240,21 @@ public class CompactionsPurgeTest String cfName = "Standard2"; ColumnFamilyStore cfs = keyspace.getColumnFamilyStore(cfName); - DecoratedKey key = Util.dk("key1"); - Mutation rm; + String key = "key1"; // inserts - rm = new Mutation(KEYSPACE1, key.getKey()); for (int i = 0; i < 5; i++) { - rm.add(cfName, cellname(String.valueOf(i)), ByteBufferUtil.EMPTY_BYTE_BUFFER, 0); + RowUpdateBuilder builder = new RowUpdateBuilder(cfs.metadata, 0, key); + builder.clustering(String.valueOf(i)) + .add("val", ByteBufferUtil.EMPTY_BYTE_BUFFER) + .build().applyUnsafe(); } - rm.applyUnsafe(); // deletes for (int i = 0; i < 5; i++) { - rm = new Mutation(KEYSPACE1, key.getKey()); - rm.delete(cfName, cellname(String.valueOf(i)), 1); - rm.applyUnsafe(); + RowUpdateBuilder.deleteRow(cfs.metadata, 1, key, String.valueOf(i)).applyUnsafe(); } cfs.forceBlockingFlush(); assertEquals(String.valueOf(cfs.getSSTables()), 1, cfs.getSSTables().size()); // inserts & deletes were in the same memtable -> only deletes in sstable @@ -265,10 +262,11 @@ public class CompactionsPurgeTest // compact and test that the row is completely gone Util.compactAll(cfs, Integer.MAX_VALUE).get(); assertTrue(cfs.getSSTables().isEmpty()); - ColumnFamily cf = keyspace.getColumnFamilyStore(cfName).getColumnFamily(QueryFilter.getIdentityFilter(key, cfName, System.currentTimeMillis())); - assertNull(String.valueOf(cf), cf); + + Util.assertEmpty(Util.cmd(cfs, key).build()); } + @Test public void testCompactionPurgeCachedRow() throws ExecutionException, InterruptedException { @@ -279,42 +277,35 @@ public class CompactionsPurgeTest Keyspace keyspace = Keyspace.open(keyspaceName); ColumnFamilyStore cfs = keyspace.getColumnFamilyStore(cfName); - DecoratedKey key = Util.dk("key3"); - Mutation rm; + String key = "key3"; // inserts - rm = new Mutation(keyspaceName, key.getKey()); for (int i = 0; i < 10; i++) { - rm.add(cfName, cellname(String.valueOf(i)), ByteBufferUtil.EMPTY_BYTE_BUFFER, 0); + RowUpdateBuilder builder = new RowUpdateBuilder(cfs.metadata, 0, key); + builder.clustering(String.valueOf(i)) + .add("val", ByteBufferUtil.EMPTY_BYTE_BUFFER) + .build().applyUnsafe(); } + + // deletes partition + Mutation rm = new Mutation(KEYSPACE_CACHED, dk(key)); + rm.add(PartitionUpdate.fullPartitionDelete(cfs.metadata, dk(key), 1, FBUtilities.nowInSeconds())); rm.applyUnsafe(); - // move the key up in row cache - cfs.getColumnFamily(QueryFilter.getIdentityFilter(key, cfName, System.currentTimeMillis())); + // Adds another unrelated partition so that the sstable is not considered fully expired. We do not + // invalidate the row cache in that latter case. + new RowUpdateBuilder(cfs.metadata, 0, "key4").clustering("c").add("val", ByteBufferUtil.EMPTY_BYTE_BUFFER).build().applyUnsafe(); - // deletes row - rm = new Mutation(keyspaceName, key.getKey()); - rm.delete(cfName, 1); - rm.applyUnsafe(); + // move the key up in row cache (it should not be empty since we have the partition deletion info) + assertFalse(Util.getOnlyPartitionUnfiltered(Util.cmd(cfs, key).build()).isEmpty()); // flush and major compact cfs.forceBlockingFlush(); Util.compactAll(cfs, Integer.MAX_VALUE).get(); - // re-inserts with timestamp lower than delete - rm = new Mutation(keyspaceName, key.getKey()); - for (int i = 0; i < 10; i++) - { - rm.add(cfName, cellname(String.valueOf(i)), ByteBufferUtil.EMPTY_BYTE_BUFFER, 0); - } - rm.applyUnsafe(); - - // Check that the second insert did went in - ColumnFamily cf = cfs.getColumnFamily(QueryFilter.getIdentityFilter(key, cfName, System.currentTimeMillis())); - assertEquals(10, cf.getColumnCount()); - for (Cell c : cf) - assertTrue(c.isLive()); + // Since we've force purging (by passing MAX_VALUE for gc_before), the row should have been invalidated and we should have no deletion info anymore + Util.assertEmpty(Util.cmd(cfs, key).build()); } @Test @@ -326,41 +317,45 @@ public class CompactionsPurgeTest String cfName = "Standard1"; Keyspace keyspace = Keyspace.open(keyspaceName); ColumnFamilyStore cfs = keyspace.getColumnFamilyStore(cfName); - DecoratedKey key = Util.dk("key3"); - Mutation rm; - QueryFilter filter = QueryFilter.getIdentityFilter(key, cfName, System.currentTimeMillis()); + String key = "key3"; // inserts - rm = new Mutation(keyspaceName, key.getKey()); for (int i = 0; i < 10; i++) - rm.add(cfName, cellname(String.valueOf(i)), ByteBufferUtil.EMPTY_BYTE_BUFFER, i); + { + RowUpdateBuilder builder = new RowUpdateBuilder(cfs.metadata, i, key); + builder.clustering(String.valueOf(i)) + .add("val", ByteBufferUtil.EMPTY_BYTE_BUFFER) + .build().applyUnsafe(); + } + + // deletes partition with timestamp such that not all columns are deleted + Mutation rm = new Mutation(KEYSPACE1, dk(key)); + rm.add(PartitionUpdate.fullPartitionDelete(cfs.metadata, dk(key), 4, FBUtilities.nowInSeconds())); rm.applyUnsafe(); - // deletes row with timestamp such that not all columns are deleted - rm = new Mutation(keyspaceName, key.getKey()); - rm.delete(cfName, 4); - rm.applyUnsafe(); - ColumnFamily cf = cfs.getColumnFamily(filter); - assertTrue(cf.isMarkedForDelete()); + ArrayBackedPartition partition = Util.getOnlyPartitionUnfiltered(Util.cmd(cfs, key).build()); + assertFalse(partition.partitionLevelDeletion().isLive()); // flush and major compact (with tombstone purging) cfs.forceBlockingFlush(); Util.compactAll(cfs, Integer.MAX_VALUE).get(); - assertFalse(cfs.getColumnFamily(filter).isMarkedForDelete()); + assertFalse(Util.getOnlyPartitionUnfiltered(Util.cmd(cfs, key).build()).isEmpty()); // re-inserts with timestamp lower than delete - rm = new Mutation(keyspaceName, key.getKey()); for (int i = 0; i < 5; i++) - rm.add(cfName, cellname(String.valueOf(i)), ByteBufferUtil.EMPTY_BYTE_BUFFER, i); - rm.applyUnsafe(); + { + RowUpdateBuilder builder = new RowUpdateBuilder(cfs.metadata, i, key); + builder.clustering(String.valueOf(i)) + .add("val", ByteBufferUtil.EMPTY_BYTE_BUFFER) + .build().applyUnsafe(); + } // Check that the second insert went in - cf = cfs.getColumnFamily(filter); - assertEquals(10, cf.getColumnCount()); - for (Cell c : cf) - assertTrue(c.isLive()); + partition = Util.getOnlyPartitionUnfiltered(Util.cmd(cfs, key).build()); + assertEquals(10, partition.rowCount()); } + @Test public void testRowTombstoneObservedBeforePurging() { @@ -370,20 +365,20 @@ public class CompactionsPurgeTest cfs.disableAutoCompaction(); // write a row out to one sstable - executeInternal(String.format("INSERT INTO %s.%s (k, v1, v2) VALUES (%d, '%s', %d)", - keyspace, table, 1, "foo", 1)); + QueryProcessor.executeInternal(String.format("INSERT INTO %s.%s (k, v1, v2) VALUES (%d, '%s', %d)", + keyspace, table, 1, "foo", 1)); cfs.forceBlockingFlush(); - UntypedResultSet result = executeInternal(String.format("SELECT * FROM %s.%s WHERE k = %d", keyspace, table, 1)); + UntypedResultSet result = QueryProcessor.executeInternal(String.format("SELECT * FROM %s.%s WHERE k = %d", keyspace, table, 1)); assertEquals(1, result.size()); // write a row tombstone out to a second sstable - executeInternal(String.format("DELETE FROM %s.%s WHERE k = %d", keyspace, table, 1)); + QueryProcessor.executeInternal(String.format("DELETE FROM %s.%s WHERE k = %d", keyspace, table, 1)); cfs.forceBlockingFlush(); // basic check that the row is considered deleted assertEquals(2, cfs.getSSTables().size()); - result = executeInternal(String.format("SELECT * FROM %s.%s WHERE k = %d", keyspace, table, 1)); + result = QueryProcessor.executeInternal(String.format("SELECT * FROM %s.%s WHERE k = %d", keyspace, table, 1)); assertEquals(0, result.size()); // compact the two sstables with a gcBefore that does *not* allow the row tombstone to be purged @@ -391,19 +386,19 @@ public class CompactionsPurgeTest // the data should be gone, but the tombstone should still exist assertEquals(1, cfs.getSSTables().size()); - result = executeInternal(String.format("SELECT * FROM %s.%s WHERE k = %d", keyspace, table, 1)); + result = QueryProcessor.executeInternal(String.format("SELECT * FROM %s.%s WHERE k = %d", keyspace, table, 1)); assertEquals(0, result.size()); // write a row out to one sstable - executeInternal(String.format("INSERT INTO %s.%s (k, v1, v2) VALUES (%d, '%s', %d)", - keyspace, table, 1, "foo", 1)); + QueryProcessor.executeInternal(String.format("INSERT INTO %s.%s (k, v1, v2) VALUES (%d, '%s', %d)", + keyspace, table, 1, "foo", 1)); cfs.forceBlockingFlush(); assertEquals(2, cfs.getSSTables().size()); - result = executeInternal(String.format("SELECT * FROM %s.%s WHERE k = %d", keyspace, table, 1)); + result = QueryProcessor.executeInternal(String.format("SELECT * FROM %s.%s WHERE k = %d", keyspace, table, 1)); assertEquals(1, result.size()); // write a row tombstone out to a different sstable - executeInternal(String.format("DELETE FROM %s.%s WHERE k = %d", keyspace, table, 1)); + QueryProcessor.executeInternal(String.format("DELETE FROM %s.%s WHERE k = %d", keyspace, table, 1)); cfs.forceBlockingFlush(); // compact the two sstables with a gcBefore that *does* allow the row tombstone to be purged @@ -411,7 +406,7 @@ public class CompactionsPurgeTest // both the data and the tombstone should be gone this time assertEquals(0, cfs.getSSTables().size()); - result = executeInternal(String.format("SELECT * FROM %s.%s WHERE k = %d", keyspace, table, 1)); + result = QueryProcessor.executeInternal(String.format("SELECT * FROM %s.%s WHERE k = %d", keyspace, table, 1)); assertEquals(0, result.size()); } } diff --git a/test/unit/org/apache/cassandra/db/compaction/CompactionsTest.java b/test/unit/org/apache/cassandra/db/compaction/CompactionsTest.java index a1fb33d70c..606d24c624 100644 --- a/test/unit/org/apache/cassandra/db/compaction/CompactionsTest.java +++ b/test/unit/org/apache/cassandra/db/compaction/CompactionsTest.java @@ -18,9 +18,6 @@ */ package org.apache.cassandra.db.compaction; -import java.io.File; -import java.io.IOException; -import java.nio.ByteBuffer; import java.util.*; import java.util.concurrent.TimeUnit; @@ -30,36 +27,26 @@ import org.apache.cassandra.Util; import org.apache.cassandra.config.KSMetaData; import org.apache.cassandra.config.CFMetaData; import org.apache.cassandra.db.*; -import org.apache.cassandra.db.columniterator.IdentityQueryFilter; -import org.apache.cassandra.db.columniterator.OnDiskAtomIterator; -import org.apache.cassandra.db.filter.QueryFilter; -import org.apache.cassandra.db.marshal.BytesType; -import org.apache.cassandra.db.marshal.LongType; +import org.apache.cassandra.db.marshal.AsciiType; import org.apache.cassandra.exceptions.ConfigurationException; import org.apache.cassandra.dht.*; -import org.apache.cassandra.io.sstable.*; import org.apache.cassandra.io.sstable.format.SSTableReader; -import org.apache.cassandra.io.sstable.format.SSTableWriter; -import org.apache.cassandra.io.sstable.metadata.StatsMetadata; import org.apache.cassandra.locator.SimpleStrategy; import org.apache.cassandra.utils.ByteBufferUtil; import org.apache.cassandra.utils.FBUtilities; -import org.apache.cassandra.utils.Pair; + import org.junit.BeforeClass; import org.junit.Ignore; import org.junit.Test; import org.junit.runner.RunWith; -import com.google.common.base.Function; -import com.google.common.collect.Iterables; -import com.google.common.collect.Sets; - import static org.junit.Assert.*; @RunWith(OrderedJUnit4ClassRunner.class) public class CompactionsTest { private static final String KEYSPACE1 = "Keyspace1"; + private static final String CF_DENSE1 = "CF_DENSE1"; private static final String CF_STANDARD1 = "CF_STANDARD1"; private static final String CF_STANDARD2 = "Standard2"; private static final String CF_STANDARD3 = "Standard3"; @@ -74,22 +61,24 @@ public class CompactionsTest Map compactionOptions = new HashMap<>(); compactionOptions.put("tombstone_compaction_interval", "1"); SchemaLoader.prepareServer(); + SchemaLoader.createKeyspace(KEYSPACE1, SimpleStrategy.class, KSMetaData.optsWithRF(1), + SchemaLoader.denseCFMD(KEYSPACE1, CF_DENSE1).compactionStrategyOptions(compactionOptions), SchemaLoader.standardCFMD(KEYSPACE1, CF_STANDARD1).compactionStrategyOptions(compactionOptions), SchemaLoader.standardCFMD(KEYSPACE1, CF_STANDARD2), SchemaLoader.standardCFMD(KEYSPACE1, CF_STANDARD3), SchemaLoader.standardCFMD(KEYSPACE1, CF_STANDARD4), - SchemaLoader.superCFMD(KEYSPACE1, CF_SUPER1, LongType.instance), - SchemaLoader.superCFMD(KEYSPACE1, CF_SUPER5, BytesType.instance), - SchemaLoader.superCFMD(KEYSPACE1, CF_SUPERGC, BytesType.instance).gcGraceSeconds(0)); + SchemaLoader.superCFMD(KEYSPACE1, CF_SUPER1, AsciiType.instance), + SchemaLoader.superCFMD(KEYSPACE1, CF_SUPER5, AsciiType.instance), + SchemaLoader.superCFMD(KEYSPACE1, CF_SUPERGC, AsciiType.instance).gcGraceSeconds(0)); } public ColumnFamilyStore testSingleSSTableCompaction(String strategyClassName) throws Exception { Keyspace keyspace = Keyspace.open(KEYSPACE1); - ColumnFamilyStore store = keyspace.getColumnFamilyStore(CF_STANDARD1); + ColumnFamilyStore store = keyspace.getColumnFamilyStore(CF_DENSE1); store.clearUnsafe(); store.metadata.gcGraceSeconds(1); store.setCompactionStrategyClass(strategyClassName); @@ -97,7 +86,7 @@ public class CompactionsTest // disable compaction while flushing store.disableAutoCompaction(); - long timestamp = populate(KEYSPACE1, CF_STANDARD1, 0, 9, 3); //ttl=3s + long timestamp = populate(KEYSPACE1, CF_DENSE1, 0, 9, 3); //ttl=3s store.forceBlockingFlush(); assertEquals(1, store.getSSTables().size()); @@ -125,29 +114,30 @@ public class CompactionsTest private long populate(String ks, String cf, int startRowKey, int endRowKey, int ttl) { long timestamp = System.currentTimeMillis(); + CFMetaData cfm = Keyspace.open(ks).getColumnFamilyStore(cf).metadata; for (int i = startRowKey; i <= endRowKey; i++) { DecoratedKey key = Util.dk(Integer.toString(i)); - Mutation rm = new Mutation(ks, key.getKey()); for (int j = 0; j < 10; j++) - rm.add(cf, Util.cellname(Integer.toString(j)), - ByteBufferUtil.EMPTY_BYTE_BUFFER, - timestamp, - j > 0 ? ttl : 0); // let first column never expire, since deleting all columns does not produce sstable - rm.applyUnsafe(); + { + new RowUpdateBuilder(cfm, timestamp, j > 0 ? ttl : 0, key.getKey()) + .clustering(Integer.toString(j)) + .add("val", ByteBufferUtil.EMPTY_BYTE_BUFFER) + .build() + .applyUnsafe(); + } } return timestamp; } - /** - * Test to see if sstable has enough expired columns, it is compacted itself. - */ + // Test to see if sstable has enough expired columns, it is compacted itself. @Test public void testSingleSSTableCompactionWithSizeTieredCompaction() throws Exception { testSingleSSTableCompaction(SizeTieredCompactionStrategy.class.getCanonicalName()); } + /* @Test public void testSingleSSTableCompactionWithLeveledCompaction() throws Exception { @@ -158,41 +148,38 @@ public class CompactionsTest } @Test - public void testSuperColumnTombstones() throws IOException + public void testSuperColumnTombstones() { Keyspace keyspace = Keyspace.open(KEYSPACE1); ColumnFamilyStore cfs = keyspace.getColumnFamilyStore("Super1"); + CFMetaData cfm = cfs.metadata; cfs.disableAutoCompaction(); DecoratedKey key = Util.dk("tskey"); ByteBuffer scName = ByteBufferUtil.bytes("TestSuperColumn"); // a subcolumn - Mutation rm = new Mutation(KEYSPACE1, key.getKey()); - rm.add("Super1", Util.cellname(scName, ByteBufferUtil.bytes(0)), - ByteBufferUtil.EMPTY_BYTE_BUFFER, - FBUtilities.timestampMicros()); - rm.applyUnsafe(); + new RowUpdateBuilder(cfm, FBUtilities.timestampMicros(), key.getKey()) + .clustering(ByteBufferUtil.bytes("cols")) + .add("val", "val1") + .build().applyUnsafe(); cfs.forceBlockingFlush(); // shadow the subcolumn with a supercolumn tombstone - rm = new Mutation(KEYSPACE1, key.getKey()); - rm.deleteRange("Super1", SuperColumns.startOf(scName), SuperColumns.endOf(scName), FBUtilities.timestampMicros()); - rm.applyUnsafe(); + RowUpdateBuilder.deleteRow(cfm, FBUtilities.timestampMicros(), key.getKey(), ByteBufferUtil.bytes("cols")).applyUnsafe(); cfs.forceBlockingFlush(); - CompactionManager.instance.performMaximal(cfs, false); + CompactionManager.instance.performMaximal(cfs); assertEquals(1, cfs.getSSTables().size()); // check that the shadowed column is gone SSTableReader sstable = cfs.getSSTables().iterator().next(); - AbstractBounds bounds = new Bounds(key, sstable.partitioner.getMinimumToken().maxKeyBound()); - ISSTableScanner scanner = sstable.getScanner(new DataRange(bounds, new IdentityQueryFilter())); - OnDiskAtomIterator iter = scanner.next(); - assertEquals(key, iter.getKey()); - assertTrue(iter.next() instanceof RangeTombstone); - assertFalse(iter.hasNext()); - scanner.close(); + AbstractBounds bounds = new Bounds(key, sstable.partitioner.getMinimumToken().maxKeyBound()); + ISSTableScanner scanner = sstable.getScanner(FBUtilities.nowInSeconds()); + UnfilteredRowIterator ai = scanner.next(); + assertTrue(ai.next() instanceof RangeTombstone); + assertFalse(ai.hasNext()); + } @Test @@ -240,9 +227,9 @@ public class CompactionsTest long newSize1 = it.next().uncompressedLength(); long newSize2 = it.next().uncompressedLength(); assertEquals("candidate sstable should not be tombstone-compacted because its key range overlap with other sstable", - originalSize1, newSize1); + originalSize1, newSize1); assertEquals("candidate sstable should not be tombstone-compacted because its key range overlap with other sstable", - originalSize2, newSize2); + originalSize2, newSize2); // now let's enable the magic property store.metadata.compactionStrategyOptions.put("unchecked_tombstone_compaction", "true"); @@ -264,6 +251,7 @@ public class CompactionsTest // make sure max timestamp of compacted sstables is recorded properly after compaction. assertMaxTimestamp(store, timestamp2); } + */ public static void assertMaxTimestamp(ColumnFamilyStore cfs, long maxTimestampExpected) { @@ -273,6 +261,7 @@ public class CompactionsTest assertEquals(maxTimestampExpected, maxTimestampObserved); } + /* @Test public void testEchoedRow() { @@ -401,24 +390,21 @@ public class CompactionsTest cf.addColumn(Util.column("a", "a", 3)); cf.deletionInfo().add(new RangeTombstone(Util.cellname("0"), Util.cellname("b"), 2, (int) (System.currentTimeMillis()/1000)),cfmeta.comparator); - try (SSTableWriter writer = SSTableWriter.create(Descriptor.fromFilename(cfs.getTempSSTablePath(dir.getDirectoryForNewSSTables())), 0, 0, 0);) - { - writer.append(Util.dk("0"), cf); - writer.append(Util.dk("1"), cf); - writer.append(Util.dk("3"), cf); + SSTableWriter writer = SSTableWriter.create(Descriptor.fromFilename(cfs.getTempSSTablePath(dir.getDirectoryForNewSSTables())), 0, 0, 0); - cfs.addSSTable(writer.finish(true)); - } - try (SSTableWriter writer = SSTableWriter.create(Descriptor.fromFilename(cfs.getTempSSTablePath(dir.getDirectoryForNewSSTables())), 0, 0, 0);) - { - writer.append(Util.dk("0"), cf); - writer.append(Util.dk("1"), cf); - writer.append(Util.dk("2"), cf); - writer.append(Util.dk("3"), cf); - cfs.addSSTable(writer.finish(true)); - } + writer.append(Util.dk("0"), cf); + writer.append(Util.dk("1"), cf); + writer.append(Util.dk("3"), cf); + cfs.addSSTable(writer.closeAndOpenReader()); + writer = SSTableWriter.create(Descriptor.fromFilename(cfs.getTempSSTablePath(dir.getDirectoryForNewSSTables())), 0, 0, 0); + + writer.append(Util.dk("0"), cf); + writer.append(Util.dk("1"), cf); + writer.append(Util.dk("2"), cf); + writer.append(Util.dk("3"), cf); + cfs.addSSTable(writer.closeAndOpenReader()); Collection toCompact = cfs.getSSTables(); assert toCompact.size() == 2; @@ -525,6 +511,7 @@ public class CompactionsTest cf = cfs.getColumnFamily(filter); assertTrue("should be empty: " + cf, cf == null || !cf.hasColumns()); } + */ private static Range rangeFor(int start, int end) { @@ -543,10 +530,16 @@ public class CompactionsTest private static void insertRowWithKey(int key) { long timestamp = System.currentTimeMillis(); - DecoratedKey decoratedKey = Util.dk(String.format("%03d", key)); + DecoratedKey dk = Util.dk(String.format("%03d", key)); + new RowUpdateBuilder(Keyspace.open(KEYSPACE1).getColumnFamilyStore(CF_STANDARD1).metadata, timestamp, dk.getKey()) + .add("val", "val") + .build() + .applyUnsafe(); + /* Mutation rm = new Mutation(KEYSPACE1, decoratedKey.getKey()); rm.add("CF_STANDARD1", Util.cellname("col"), ByteBufferUtil.EMPTY_BYTE_BUFFER, timestamp, 1000); rm.applyUnsafe(); + */ } @Test diff --git a/test/unit/org/apache/cassandra/db/compaction/DateTieredCompactionStrategyTest.java b/test/unit/org/apache/cassandra/db/compaction/DateTieredCompactionStrategyTest.java index 64e44654b7..cbacfba91d 100644 --- a/test/unit/org/apache/cassandra/db/compaction/DateTieredCompactionStrategyTest.java +++ b/test/unit/org/apache/cassandra/db/compaction/DateTieredCompactionStrategyTest.java @@ -34,6 +34,7 @@ import org.apache.cassandra.db.ColumnFamilyStore; import org.apache.cassandra.db.DecoratedKey; import org.apache.cassandra.db.Keyspace; import org.apache.cassandra.db.Mutation; +import org.apache.cassandra.db.RowUpdateBuilder; import org.apache.cassandra.exceptions.ConfigurationException; import org.apache.cassandra.io.sstable.format.SSTableReader; import org.apache.cassandra.locator.SimpleStrategy; @@ -139,10 +140,10 @@ public class DateTieredCompactionStrategyTest extends SchemaLoader public void testGetBuckets() { List> pairs = Lists.newArrayList( - Pair.create("a", 199L), - Pair.create("b", 299L), - Pair.create("a", 1L), - Pair.create("b", 201L) + Pair.create("a", 199L), + Pair.create("b", 299L), + Pair.create("a", 1L), + Pair.create("b", 201L) ); List> buckets = getBuckets(pairs, 100L, 2, 200L); assertEquals(2, buckets.size()); @@ -215,9 +216,10 @@ public class DateTieredCompactionStrategyTest extends SchemaLoader for (int r = 0; r < numSSTables; r++) { DecoratedKey key = Util.dk(String.valueOf(r)); - Mutation rm = new Mutation(KEYSPACE1, key.getKey()); - rm.add(CF_STANDARD1, Util.cellname("column"), value, r); - rm.apply(); + new RowUpdateBuilder(cfs.metadata, r, key.getKey()) + .clustering("column") + .add("val", value).build().applyUnsafe(); + cfs.forceBlockingFlush(); } cfs.forceBlockingFlush(); @@ -262,9 +264,10 @@ public class DateTieredCompactionStrategyTest extends SchemaLoader for (int r = 0; r < numSSTables; r++) { DecoratedKey key = Util.dk(String.valueOf(r)); - Mutation rm = new Mutation(KEYSPACE1, key.getKey()); - rm.add(CF_STANDARD1, Util.cellname("column"), value, r); - rm.apply(); + new RowUpdateBuilder(cfs.metadata, r, key.getKey()) + .clustering("column") + .add("val", value).build().applyUnsafe(); + cfs.forceBlockingFlush(); } cfs.forceBlockingFlush(); @@ -298,16 +301,19 @@ public class DateTieredCompactionStrategyTest extends SchemaLoader // create 2 sstables DecoratedKey key = Util.dk(String.valueOf("expired")); - Mutation rm = new Mutation(KEYSPACE1, key.getKey()); - rm.add(CF_STANDARD1, Util.cellname("column"), value, System.currentTimeMillis(), 1); - rm.apply(); + new RowUpdateBuilder(cfs.metadata, System.currentTimeMillis(), 1, key.getKey()) + .clustering("column") + .add("val", value).build().applyUnsafe(); + cfs.forceBlockingFlush(); SSTableReader expiredSSTable = cfs.getSSTables().iterator().next(); Thread.sleep(10); + key = Util.dk(String.valueOf("nonexpired")); - rm = new Mutation(KEYSPACE1, key.getKey()); - rm.add(CF_STANDARD1, Util.cellname("column"), value, System.currentTimeMillis()); - rm.apply(); + new RowUpdateBuilder(cfs.metadata, System.currentTimeMillis(), key.getKey()) + .clustering("column") + .add("val", value).build().applyUnsafe(); + cfs.forceBlockingFlush(); assertEquals(cfs.getSSTables().size(), 2); diff --git a/test/unit/org/apache/cassandra/db/compaction/LeveledCompactionStrategyTest.java b/test/unit/org/apache/cassandra/db/compaction/LeveledCompactionStrategyTest.java index d85bd6a7d9..a331afad5c 100644 --- a/test/unit/org/apache/cassandra/db/compaction/LeveledCompactionStrategyTest.java +++ b/test/unit/org/apache/cassandra/db/compaction/LeveledCompactionStrategyTest.java @@ -1,4 +1,4 @@ -/** +/* * Licensed to the Apache Software Foundation (ASF) under one * or more contributor license agreements. See the NOTICE file * distributed with this work for additional information @@ -27,7 +27,6 @@ import java.util.Map; import java.util.Random; import java.util.UUID; -import org.apache.cassandra.io.sstable.format.SSTableReader; import org.junit.After; import org.junit.Before; import org.junit.BeforeClass; @@ -37,6 +36,7 @@ import org.junit.runner.RunWith; import org.apache.cassandra.OrderedJUnit4ClassRunner; import org.apache.cassandra.SchemaLoader; import org.apache.cassandra.Util; +import org.apache.cassandra.UpdateBuilder; import org.apache.cassandra.config.KSMetaData; import org.apache.cassandra.db.ColumnFamilyStore; import org.apache.cassandra.db.DecoratedKey; @@ -46,6 +46,7 @@ import org.apache.cassandra.dht.Range; import org.apache.cassandra.dht.Token; import org.apache.cassandra.exceptions.ConfigurationException; import org.apache.cassandra.io.sstable.ISSTableScanner; +import org.apache.cassandra.io.sstable.format.SSTableReader; import org.apache.cassandra.locator.SimpleStrategy; import org.apache.cassandra.notifications.SSTableAddedNotification; import org.apache.cassandra.notifications.SSTableRepairStatusChanged; @@ -114,13 +115,10 @@ public class LeveledCompactionStrategyTest // Adds enough data to trigger multiple sstable per level for (int r = 0; r < rows; r++) { - DecoratedKey key = Util.dk(String.valueOf(r)); - Mutation rm = new Mutation(KEYSPACE1, key.getKey()); + UpdateBuilder update = UpdateBuilder.create(cfs.metadata, String.valueOf(r)); for (int c = 0; c < columns; c++) - { - rm.add(CF_STANDARDDLEVELED, Util.cellname("column" + c), value, 0); - } - rm.apply(); + update.newRow("column" + c).add("val", value); + update.applyUnsafe(); cfs.forceBlockingFlush(); } @@ -165,13 +163,10 @@ public class LeveledCompactionStrategyTest // Adds enough data to trigger multiple sstable per level for (int r = 0; r < rows; r++) { - DecoratedKey key = Util.dk(String.valueOf(r)); - Mutation rm = new Mutation(KEYSPACE1, key.getKey()); + UpdateBuilder update = UpdateBuilder.create(cfs.metadata, String.valueOf(r)); for (int c = 0; c < columns; c++) - { - rm.add(CF_STANDARDDLEVELED, Util.cellname("column" + c), value, 0); - } - rm.applyUnsafe(); + update.newRow("column" + c).add("val", value); + update.applyUnsafe(); cfs.forceBlockingFlush(); } @@ -182,7 +177,7 @@ public class LeveledCompactionStrategyTest assertTrue(strategy.getSSTableCountPerLevel()[2] > 0); Range range = new Range<>(Util.token(""), Util.token("")); - int gcBefore = keyspace.getColumnFamilyStore(CF_STANDARDDLEVELED).gcBefore(System.currentTimeMillis()); + int gcBefore = keyspace.getColumnFamilyStore(CF_STANDARDDLEVELED).gcBefore(FBUtilities.nowInSeconds()); UUID parentRepSession = UUID.randomUUID(); ActiveRepairService.instance.registerParentRepairSession(parentRepSession, Arrays.asList(cfs), Arrays.asList(range), false); RepairJobDesc desc = new RepairJobDesc(parentRepSession, UUID.randomUUID(), KEYSPACE1, CF_STANDARDDLEVELED, range); @@ -212,13 +207,10 @@ public class LeveledCompactionStrategyTest int columns = 10; for (int r = 0; r < rows; r++) { - DecoratedKey key = Util.dk(String.valueOf(r)); - Mutation rm = new Mutation(KEYSPACE1, key.getKey()); + UpdateBuilder update = UpdateBuilder.create(cfs.metadata, String.valueOf(r)); for (int c = 0; c < columns; c++) - { - rm.add(CF_STANDARDDLEVELED, Util.cellname("column" + c), value, 0); - } - rm.applyUnsafe(); + update.newRow("column" + c).add("val", value); + update.applyUnsafe(); cfs.forceBlockingFlush(); } @@ -251,13 +243,10 @@ public class LeveledCompactionStrategyTest // Adds enough data to trigger multiple sstable per level for (int r = 0; r < rows; r++) { - DecoratedKey key = Util.dk(String.valueOf(r)); - Mutation rm = new Mutation(KEYSPACE1, key.getKey()); + UpdateBuilder update = UpdateBuilder.create(cfs.metadata, String.valueOf(r)); for (int c = 0; c < columns; c++) - { - rm.add(CF_STANDARDDLEVELED, Util.cellname("column" + c), value, 0); - } - rm.applyUnsafe(); + update.newRow("column" + c).add("val", value); + update.applyUnsafe(); cfs.forceBlockingFlush(); } waitForLeveling(cfs); @@ -299,13 +288,10 @@ public class LeveledCompactionStrategyTest // Adds enough data to trigger multiple sstable per level for (int r = 0; r < rows; r++) { - DecoratedKey key = Util.dk(String.valueOf(r)); - Mutation rm = new Mutation(KEYSPACE1, key.getKey()); + UpdateBuilder update = UpdateBuilder.create(cfs.metadata, String.valueOf(r)); for (int c = 0; c < columns; c++) - { - rm.add(CF_STANDARDDLEVELED, Util.cellname("column" + c), value, 0); - } - rm.applyUnsafe(); + update.newRow("column" + c).add("val", value); + update.applyUnsafe(); cfs.forceBlockingFlush(); } waitForLeveling(cfs); diff --git a/test/unit/org/apache/cassandra/db/compaction/OneCompactionTest.java b/test/unit/org/apache/cassandra/db/compaction/OneCompactionTest.java index ec5c280128..a775883253 100644 --- a/test/unit/org/apache/cassandra/db/compaction/OneCompactionTest.java +++ b/test/unit/org/apache/cassandra/db/compaction/OneCompactionTest.java @@ -1,44 +1,44 @@ /* -* Licensed to the Apache Software Foundation (ASF) under one -* or more contributor license agreements. See the NOTICE file -* distributed with this work for additional information -* regarding copyright ownership. The ASF licenses this file -* to you under the Apache License, Version 2.0 (the -* "License"); you may not use this file except in compliance -* with the License. You may obtain a copy of the License at -* -* http://www.apache.org/licenses/LICENSE-2.0 -* -* Unless required by applicable law or agreed to in writing, -* software distributed under the License is distributed on an -* "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -* KIND, either express or implied. See the License for the -* specific language governing permissions and limitations -* under the License. -*/ + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ package org.apache.cassandra.db.compaction; import java.util.HashMap; -import java.util.Map; -import java.util.concurrent.ExecutionException; -import java.util.Set; import java.util.HashSet; +import java.util.Map; +import java.util.Set; -import org.apache.cassandra.Util; - +import com.google.common.collect.Iterables; import org.junit.BeforeClass; import org.junit.Test; -import static org.junit.Assert.assertEquals; - -import org.apache.cassandra.db.*; - import org.apache.cassandra.SchemaLoader; +import org.apache.cassandra.Util; import org.apache.cassandra.config.KSMetaData; +import org.apache.cassandra.db.ColumnFamilyStore; +import org.apache.cassandra.db.Keyspace; +import org.apache.cassandra.db.RowUpdateBuilder; +import org.apache.cassandra.db.partitions.UnfilteredPartitionIterator; import org.apache.cassandra.exceptions.ConfigurationException; import org.apache.cassandra.locator.SimpleStrategy; import org.apache.cassandra.utils.ByteBufferUtil; +import static org.junit.Assert.assertEquals; + public class OneCompactionTest { @@ -66,15 +66,18 @@ public class OneCompactionTest Keyspace keyspace = Keyspace.open(KEYSPACE1); ColumnFamilyStore store = keyspace.getColumnFamilyStore(columnFamilyName); - Set inserted = new HashSet(); + Set inserted = new HashSet<>(); for (int j = 0; j < insertsPerTable; j++) { - DecoratedKey key = Util.dk(String.valueOf(j)); - Mutation rm = new Mutation(KEYSPACE1, key.getKey()); - rm.add(columnFamilyName, Util.cellname("0"), ByteBufferUtil.EMPTY_BYTE_BUFFER, j); - rm.applyUnsafe(); + String key = String.valueOf(j); + new RowUpdateBuilder(store.metadata, j, key) + .clustering("0") + .add("val", ByteBufferUtil.EMPTY_BYTE_BUFFER) + .build() + .applyUnsafe(); + inserted.add(key); store.forceBlockingFlush(); - assertEquals(inserted.size(), Util.getRangeSlice(store).size()); + assertEquals(inserted.size(), Util.getAll(Util.cmd(store).build()).size()); } CompactionManager.instance.performMaximal(store, false); assertEquals(1, store.getSSTables().size()); diff --git a/test/unit/org/apache/cassandra/db/compaction/SizeTieredCompactionStrategyTest.java b/test/unit/org/apache/cassandra/db/compaction/SizeTieredCompactionStrategyTest.java index 46d54d9740..e70d9a85cc 100644 --- a/test/unit/org/apache/cassandra/db/compaction/SizeTieredCompactionStrategyTest.java +++ b/test/unit/org/apache/cassandra/db/compaction/SizeTieredCompactionStrategyTest.java @@ -1,4 +1,4 @@ -/** +/* * Licensed to the Apache Software Foundation (ASF) under one * or more contributor license agreements. See the NOTICE file * distributed with this work for additional information @@ -18,17 +18,25 @@ package org.apache.cassandra.db.compaction; import java.nio.ByteBuffer; -import java.util.*; +import java.util.ArrayList; +import java.util.Collections; +import java.util.HashMap; +import java.util.List; +import java.util.Map; -import org.apache.cassandra.io.sstable.format.SSTableReader; import org.junit.BeforeClass; import org.junit.Test; import org.apache.cassandra.SchemaLoader; import org.apache.cassandra.Util; import org.apache.cassandra.config.KSMetaData; -import org.apache.cassandra.db.*; +import org.apache.cassandra.db.ColumnFamilyStore; +import org.apache.cassandra.db.DecoratedKey; +import org.apache.cassandra.db.Keyspace; +import org.apache.cassandra.db.Mutation; +import org.apache.cassandra.db.RowUpdateBuilder; import org.apache.cassandra.exceptions.ConfigurationException; +import org.apache.cassandra.io.sstable.format.SSTableReader; import org.apache.cassandra.locator.SimpleStrategy; import org.apache.cassandra.metrics.RestorableMeter; import org.apache.cassandra.utils.Pair; @@ -37,8 +45,9 @@ import static org.apache.cassandra.db.compaction.SizeTieredCompactionStrategy.ge import static org.apache.cassandra.db.compaction.SizeTieredCompactionStrategy.mostInterestingBucket; import static org.apache.cassandra.db.compaction.SizeTieredCompactionStrategy.trimToThresholdWithHotness; import static org.apache.cassandra.db.compaction.SizeTieredCompactionStrategy.validateOptions; - -import static org.junit.Assert.*; +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertTrue; +import static org.junit.Assert.fail; public class SizeTieredCompactionStrategyTest { @@ -156,10 +165,10 @@ public class SizeTieredCompactionStrategyTest int numSSTables = 3; for (int r = 0; r < numSSTables; r++) { - DecoratedKey key = Util.dk(String.valueOf(r)); - Mutation rm = new Mutation(ksname, key.getKey()); - rm.add(cfname, Util.cellname("column"), value, 0); - rm.applyUnsafe(); + String key = String.valueOf(r); + new RowUpdateBuilder(cfs.metadata, 0, key) + .clustering("column").add("val", value) + .build().applyUnsafe(); cfs.forceBlockingFlush(); } cfs.forceBlockingFlush(); diff --git a/test/unit/org/apache/cassandra/db/compaction/TTLExpiryTest.java b/test/unit/org/apache/cassandra/db/compaction/TTLExpiryTest.java index 579794dfd2..eae11b6a50 100644 --- a/test/unit/org/apache/cassandra/db/compaction/TTLExpiryTest.java +++ b/test/unit/org/apache/cassandra/db/compaction/TTLExpiryTest.java @@ -20,6 +20,10 @@ package org.apache.cassandra.db.compaction; * */ +import org.apache.cassandra.config.CFMetaData; +import org.apache.cassandra.cql3.ColumnIdentifier; +import org.apache.cassandra.db.rows.UnfilteredRowIterator; +import org.apache.cassandra.db.marshal.AsciiType; import org.apache.cassandra.io.sstable.format.SSTableReader; import org.junit.BeforeClass; import com.google.common.collect.Sets; @@ -31,11 +35,12 @@ import org.apache.cassandra.SchemaLoader; import org.apache.cassandra.Util; import org.apache.cassandra.config.KSMetaData; import org.apache.cassandra.db.*; -import org.apache.cassandra.db.columniterator.OnDiskAtomIterator; +import org.apache.cassandra.db.filter.ColumnFilter; import org.apache.cassandra.exceptions.ConfigurationException; import org.apache.cassandra.io.sstable.ISSTableScanner; import org.apache.cassandra.locator.SimpleStrategy; import org.apache.cassandra.utils.ByteBufferUtil; +import org.apache.cassandra.utils.FBUtilities; import java.io.IOException; import java.util.Collections; @@ -55,9 +60,18 @@ public class TTLExpiryTest { SchemaLoader.prepareServer(); SchemaLoader.createKeyspace(KEYSPACE1, - SimpleStrategy.class, - KSMetaData.optsWithRF(1), - SchemaLoader.standardCFMD(KEYSPACE1, CF_STANDARD1)); + SimpleStrategy.class, + KSMetaData.optsWithRF(1), + CFMetaData.Builder.create(KEYSPACE1, CF_STANDARD1) + .addPartitionKey("pKey", AsciiType.instance) + .addRegularColumn("col1", AsciiType.instance) + .addRegularColumn("col", AsciiType.instance) + .addRegularColumn("col311", AsciiType.instance) + .addRegularColumn("col2", AsciiType.instance) + .addRegularColumn("col3", AsciiType.instance) + .addRegularColumn("col7", AsciiType.instance) + .addRegularColumn("shadow", AsciiType.instance) + .build().gcGraceSeconds(0)); } @Test @@ -66,30 +80,52 @@ public class TTLExpiryTest ColumnFamilyStore cfs = Keyspace.open(KEYSPACE1).getColumnFamilyStore("Standard1"); cfs.disableAutoCompaction(); cfs.metadata.gcGraceSeconds(0); + String key = "ttl"; + new RowUpdateBuilder(cfs.metadata, 1L, 1, key) + .add("col1", ByteBufferUtil.EMPTY_BYTE_BUFFER) + .build() + .applyUnsafe(); + + new RowUpdateBuilder(cfs.metadata, 3L, 1, key) + .add("col2", ByteBufferUtil.EMPTY_BYTE_BUFFER) + .build() + .applyUnsafe(); + cfs.forceBlockingFlush(); + new RowUpdateBuilder(cfs.metadata, 2L, 1, key) + .add("col1", ByteBufferUtil.EMPTY_BYTE_BUFFER) + .build() + .applyUnsafe(); + + new RowUpdateBuilder(cfs.metadata, 5L, 1, key) + .add("col2", ByteBufferUtil.EMPTY_BYTE_BUFFER) + .build() + .applyUnsafe(); - DecoratedKey ttlKey = Util.dk("ttl"); - Mutation rm = new Mutation(KEYSPACE1, ttlKey.getKey()); - rm.add("Standard1", Util.cellname("col1"), ByteBufferUtil.EMPTY_BYTE_BUFFER, 1, 1); - rm.add("Standard1", Util.cellname("col2"), ByteBufferUtil.EMPTY_BYTE_BUFFER, 3, 1); - rm.applyUnsafe(); cfs.forceBlockingFlush(); - rm = new Mutation(KEYSPACE1, ttlKey.getKey()); - rm.add("Standard1", Util.cellname("col1"), ByteBufferUtil.EMPTY_BYTE_BUFFER, 2, 1); - rm.add("Standard1", Util.cellname("col2"), ByteBufferUtil.EMPTY_BYTE_BUFFER, 5, 1); - rm.applyUnsafe(); + new RowUpdateBuilder(cfs.metadata, 4L, 1, key) + .add("col1", ByteBufferUtil.EMPTY_BYTE_BUFFER) + .build() + .applyUnsafe(); + + new RowUpdateBuilder(cfs.metadata, 7L, 1, key) + .add("shadow", ByteBufferUtil.EMPTY_BYTE_BUFFER) + .build() + .applyUnsafe(); + cfs.forceBlockingFlush(); - rm = new Mutation(KEYSPACE1, ttlKey.getKey()); - rm.add("Standard1", Util.cellname("col1"), ByteBufferUtil.EMPTY_BYTE_BUFFER, 4, 1); - rm.add("Standard1", Util.cellname("shadow"), ByteBufferUtil.EMPTY_BYTE_BUFFER, 7, 1); - rm.applyUnsafe(); - cfs.forceBlockingFlush(); - rm = new Mutation(KEYSPACE1, ttlKey.getKey()); - rm.add("Standard1", Util.cellname("shadow"), ByteBufferUtil.EMPTY_BYTE_BUFFER, 6, 3); - rm.add("Standard1", Util.cellname("col2"), ByteBufferUtil.EMPTY_BYTE_BUFFER, 8, 1); - rm.applyUnsafe(); + new RowUpdateBuilder(cfs.metadata, 6L, 3, key) + .add("shadow", ByteBufferUtil.EMPTY_BYTE_BUFFER) + .build() + .applyUnsafe(); + + new RowUpdateBuilder(cfs.metadata, 8L, 1, key) + .add("col2", ByteBufferUtil.EMPTY_BYTE_BUFFER) + .build() + .applyUnsafe(); + cfs.forceBlockingFlush(); Set sstables = Sets.newHashSet(cfs.getSSTables()); @@ -112,39 +148,34 @@ public class TTLExpiryTest cfs.disableAutoCompaction(); cfs.metadata.gcGraceSeconds(0); long timestamp = System.currentTimeMillis(); - Mutation rm = new Mutation(KEYSPACE1, Util.dk("ttl").getKey()); - rm.add("Standard1", Util.cellname("col"), - ByteBufferUtil.EMPTY_BYTE_BUFFER, - timestamp, - 1); - rm.add("Standard1", Util.cellname("col7"), - ByteBufferUtil.EMPTY_BYTE_BUFFER, - timestamp, - 1); + String key = "ttl"; + new RowUpdateBuilder(cfs.metadata, timestamp, 1, key) + .add("col", ByteBufferUtil.EMPTY_BYTE_BUFFER) + .add("col7", ByteBufferUtil.EMPTY_BYTE_BUFFER) + .build() + .applyUnsafe(); - rm.applyUnsafe(); cfs.forceBlockingFlush(); - rm = new Mutation(KEYSPACE1, Util.dk("ttl").getKey()); - rm.add("Standard1", Util.cellname("col2"), - ByteBufferUtil.EMPTY_BYTE_BUFFER, - timestamp, - 1); - rm.applyUnsafe(); + new RowUpdateBuilder(cfs.metadata, timestamp, 1, key) + .add("col2", ByteBufferUtil.EMPTY_BYTE_BUFFER) + .build() + .applyUnsafe(); + + cfs.forceBlockingFlush(); - rm = new Mutation(KEYSPACE1, Util.dk("ttl").getKey()); - rm.add("Standard1", Util.cellname("col3"), - ByteBufferUtil.EMPTY_BYTE_BUFFER, - timestamp, - 1); - rm.applyUnsafe(); + new RowUpdateBuilder(cfs.metadata, timestamp, 1, key) + .add("col3", ByteBufferUtil.EMPTY_BYTE_BUFFER) + .build() + .applyUnsafe(); + + cfs.forceBlockingFlush(); - rm = new Mutation(KEYSPACE1, Util.dk("ttl").getKey()); - rm.add("Standard1", Util.cellname("col311"), - ByteBufferUtil.EMPTY_BYTE_BUFFER, - timestamp, - 1); - rm.applyUnsafe(); + new RowUpdateBuilder(cfs.metadata, timestamp, 1, key) + .add("col311", ByteBufferUtil.EMPTY_BYTE_BUFFER) + .build() + .applyUnsafe(); + cfs.forceBlockingFlush(); Thread.sleep(2000); // wait for ttl to expire @@ -160,53 +191,43 @@ public class TTLExpiryTest cfs.disableAutoCompaction(); cfs.metadata.gcGraceSeconds(0); long timestamp = System.currentTimeMillis(); - Mutation rm = new Mutation(KEYSPACE1, Util.dk("ttl").getKey()); - rm.add("Standard1", Util.cellname("col"), - ByteBufferUtil.EMPTY_BYTE_BUFFER, - timestamp, - 1); - rm.add("Standard1", Util.cellname("col7"), - ByteBufferUtil.EMPTY_BYTE_BUFFER, - timestamp, - 1); + String key = "ttl"; + new RowUpdateBuilder(cfs.metadata, timestamp, 1, key) + .add("col", ByteBufferUtil.EMPTY_BYTE_BUFFER) + .add("col7", ByteBufferUtil.EMPTY_BYTE_BUFFER) + .build() + .applyUnsafe(); - rm.applyUnsafe(); cfs.forceBlockingFlush(); + new RowUpdateBuilder(cfs.metadata, timestamp, 1, key) + .add("col2", ByteBufferUtil.EMPTY_BYTE_BUFFER) + .build() + .applyUnsafe(); + cfs.forceBlockingFlush(); + new RowUpdateBuilder(cfs.metadata, timestamp, 1, key) + .add("col3", ByteBufferUtil.EMPTY_BYTE_BUFFER) + .build() + .applyUnsafe(); + cfs.forceBlockingFlush(); + String noTTLKey = "nottl"; + new RowUpdateBuilder(cfs.metadata, timestamp, noTTLKey) + .add("col311", ByteBufferUtil.EMPTY_BYTE_BUFFER) + .build() + .applyUnsafe(); - rm = new Mutation(KEYSPACE1, Util.dk("ttl").getKey()); - rm.add("Standard1", Util.cellname("col2"), - ByteBufferUtil.EMPTY_BYTE_BUFFER, - timestamp, - 1); - rm.applyUnsafe(); - cfs.forceBlockingFlush(); - rm = new Mutation(KEYSPACE1, Util.dk("ttl").getKey()); - rm.add("Standard1", Util.cellname("col3"), - ByteBufferUtil.EMPTY_BYTE_BUFFER, - timestamp, - 1); - rm.applyUnsafe(); - cfs.forceBlockingFlush(); - DecoratedKey noTTLKey = Util.dk("nottl"); - rm = new Mutation(KEYSPACE1, noTTLKey.getKey()); - rm.add("Standard1", Util.cellname("col311"), - ByteBufferUtil.EMPTY_BYTE_BUFFER, - timestamp); - rm.applyUnsafe(); cfs.forceBlockingFlush(); Thread.sleep(2000); // wait for ttl to expire assertEquals(4, cfs.getSSTables().size()); cfs.enableAutoCompaction(true); assertEquals(1, cfs.getSSTables().size()); SSTableReader sstable = cfs.getSSTables().iterator().next(); - ISSTableScanner scanner = sstable.getScanner(DataRange.allData(sstable.partitioner)); + ISSTableScanner scanner = sstable.getScanner(ColumnFilter.all(sstable.metadata), DataRange.allData(sstable.partitioner), false); assertTrue(scanner.hasNext()); while(scanner.hasNext()) { - OnDiskAtomIterator iter = scanner.next(); - assertEquals(noTTLKey, iter.getKey()); + UnfilteredRowIterator iter = scanner.next(); + assertEquals(Util.dk(noTTLKey), iter.partitionKey()); } - scanner.close(); } } diff --git a/test/unit/org/apache/cassandra/db/composites/CTypeTest.java b/test/unit/org/apache/cassandra/db/composites/CTypeTest.java index 496a2dc4f7..9b261e6a2a 100644 --- a/test/unit/org/apache/cassandra/db/composites/CTypeTest.java +++ b/test/unit/org/apache/cassandra/db/composites/CTypeTest.java @@ -17,98 +17,113 @@ */ package org.apache.cassandra.db.composites; -import com.google.common.collect.Lists; import org.apache.cassandra.db.marshal.*; import org.apache.cassandra.utils.ByteBufferUtil; import org.junit.Test; -import java.util.List; +import java.nio.ByteBuffer; public class CTypeTest { - static final List> types = Lists.newArrayList(); - static - { - types.add(UTF8Type.instance); - types.add(UUIDType.instance); - types.add(Int32Type.instance); - } - - static final CellNameType cdtype = new CompoundDenseCellNameType(types); - static final CellNameType stype1 = new SimpleDenseCellNameType(BytesType.instance); - static final CellNameType stype2 = new SimpleDenseCellNameType(UUIDType.instance); - @Test public void testCompoundType() { - Composite a1 = cdtype.makeCellName("a",UUIDType.instance.fromString("00000000-0000-0000-0000-000000000000"), 1); - Composite a2 = cdtype.makeCellName("a",UUIDType.instance.fromString("00000000-0000-0000-0000-000000000000"), 100); - Composite b1 = cdtype.makeCellName("a",UUIDType.instance.fromString("ffffffff-ffff-ffff-ffff-ffffffffffff"), 1); - Composite b2 = cdtype.makeCellName("a",UUIDType.instance.fromString("ffffffff-ffff-ffff-ffff-ffffffffffff"), 100); - Composite c1 = cdtype.makeCellName("z",UUIDType.instance.fromString("00000000-0000-0000-0000-000000000000"), 1); - Composite c2 = cdtype.makeCellName("z",UUIDType.instance.fromString("00000000-0000-0000-0000-000000000000"), 100); - Composite d1 = cdtype.makeCellName("z",UUIDType.instance.fromString("ffffffff-ffff-ffff-ffff-ffffffffffff"), 1); - Composite d2 = cdtype.makeCellName("z",UUIDType.instance.fromString("ffffffff-ffff-ffff-ffff-ffffffffffff"), 100); + CompositeType baseType = CompositeType.getInstance(AsciiType.instance, UUIDType.instance, LongType.instance); - Composite z1 = cdtype.makeCellName(ByteBufferUtil.EMPTY_BYTE_BUFFER,UUIDType.instance.fromString("ffffffff-ffff-ffff-ffff-ffffffffffff"), 100); + ByteBuffer a1 = baseType.builder() + .add(ByteBufferUtil.bytes("a")) + .add(UUIDType.instance.fromString("00000000-0000-0000-0000-000000000000")) + .add(ByteBufferUtil.bytes(1)).build(); + ByteBuffer a2 = baseType.builder() + .add(ByteBufferUtil.bytes("a")) + .add(UUIDType.instance.fromString("00000000-0000-0000-0000-000000000000")) + .add(ByteBufferUtil.bytes(100)).build(); + ByteBuffer b1 = baseType.builder() + .add(ByteBufferUtil.bytes("a")) + .add(UUIDType.instance.fromString("ffffffff-ffff-ffff-ffff-ffffffffffff")) + .add(ByteBufferUtil.bytes(1)).build(); + ByteBuffer b2 = baseType.builder() + .add(ByteBufferUtil.bytes("a")) + .add(UUIDType.instance.fromString("ffffffff-ffff-ffff-ffff-ffffffffffff")) + .add(ByteBufferUtil.bytes(100)).build(); + ByteBuffer c1 = baseType.builder() + .add(ByteBufferUtil.bytes("z")) + .add(UUIDType.instance.fromString("00000000-0000-0000-0000-000000000000")) + .add(ByteBufferUtil.bytes(1)).build(); + ByteBuffer c2 = baseType.builder() + .add(ByteBufferUtil.bytes("z")) + .add(UUIDType.instance.fromString("00000000-0000-0000-0000-000000000000")) + .add(ByteBufferUtil.bytes(100)).build(); + ByteBuffer d1 = baseType.builder() + .add(ByteBufferUtil.bytes("z")) + .add(UUIDType.instance.fromString("ffffffff-ffff-ffff-ffff-ffffffffffff")) + .add(ByteBufferUtil.bytes(1)).build(); + ByteBuffer d2 = baseType.builder() + .add(ByteBufferUtil.bytes("z")) + .add(UUIDType.instance.fromString("ffffffff-ffff-ffff-ffff-ffffffffffff")) + .add(ByteBufferUtil.bytes(100)).build(); + ByteBuffer z1 = baseType.builder() + .add(ByteBufferUtil.EMPTY_BYTE_BUFFER) + .add(UUIDType.instance.fromString("ffffffff-ffff-ffff-ffff-ffffffffffff")) + .add(ByteBufferUtil.bytes(100)).build(); - assert cdtype.compare(a1,a2) < 0; - assert cdtype.compare(a2,b1) < 0; - assert cdtype.compare(b1,b2) < 0; - assert cdtype.compare(b2,c1) < 0; - assert cdtype.compare(c1,c2) < 0; - assert cdtype.compare(c2,d1) < 0; - assert cdtype.compare(d1,d2) < 0; + assert baseType.compare(a1,a2) < 0; + assert baseType.compare(a2,b1) < 0; + assert baseType.compare(b1,b2) < 0; + assert baseType.compare(b2,c1) < 0; + assert baseType.compare(c1,c2) < 0; + assert baseType.compare(c2,d1) < 0; + assert baseType.compare(d1,d2) < 0; - assert cdtype.compare(a2,a1) > 0; - assert cdtype.compare(b1,a2) > 0; - assert cdtype.compare(b2,b1) > 0; - assert cdtype.compare(c1,b2) > 0; - assert cdtype.compare(c2,c1) > 0; - assert cdtype.compare(d1,c2) > 0; - assert cdtype.compare(d2,d1) > 0; + assert baseType.compare(a2,a1) > 0; + assert baseType.compare(b1,a2) > 0; + assert baseType.compare(b2,b1) > 0; + assert baseType.compare(c1,b2) > 0; + assert baseType.compare(c2,c1) > 0; + assert baseType.compare(d1,c2) > 0; + assert baseType.compare(d2,d1) > 0; - assert cdtype.compare(z1,a1) < 0; - assert cdtype.compare(z1,a2) < 0; - assert cdtype.compare(z1,b1) < 0; - assert cdtype.compare(z1,b2) < 0; - assert cdtype.compare(z1,c1) < 0; - assert cdtype.compare(z1,c2) < 0; - assert cdtype.compare(z1,d1) < 0; - assert cdtype.compare(z1,d2) < 0; + assert baseType.compare(z1,a1) < 0; + assert baseType.compare(z1,a2) < 0; + assert baseType.compare(z1,b1) < 0; + assert baseType.compare(z1,b2) < 0; + assert baseType.compare(z1,c1) < 0; + assert baseType.compare(z1,c2) < 0; + assert baseType.compare(z1,d1) < 0; + assert baseType.compare(z1,d2) < 0; - assert cdtype.compare(a1,a1) == 0; - assert cdtype.compare(a2,a2) == 0; - assert cdtype.compare(b1,b1) == 0; - assert cdtype.compare(b2,b2) == 0; - assert cdtype.compare(c1,c1) == 0; - assert cdtype.compare(c2,c2) == 0; - assert cdtype.compare(z1,z1) == 0; + assert baseType.compare(a1,a1) == 0; + assert baseType.compare(a2,a2) == 0; + assert baseType.compare(b1,b1) == 0; + assert baseType.compare(b2,b2) == 0; + assert baseType.compare(c1,c1) == 0; + assert baseType.compare(c2,c2) == 0; + assert baseType.compare(z1,z1) == 0; } @Test public void testSimpleType2() { - CellName a = stype2.makeCellName(UUIDType.instance.fromString("00000000-0000-0000-0000-000000000000")); - CellName z = stype2.makeCellName(UUIDType.instance.fromString("ffffffff-ffff-ffff-ffff-ffffffffffff")); + CompositeType baseType = CompositeType.getInstance(UUIDType.instance); + ByteBuffer a = baseType.builder().add(UUIDType.instance.fromString("00000000-0000-0000-0000-000000000000")).build(); + ByteBuffer z = baseType.builder().add(UUIDType.instance.fromString("ffffffff-ffff-ffff-ffff-ffffffffffff")).build(); - assert stype2.compare(a,z) < 0; - assert stype2.compare(z,a) > 0; - assert stype2.compare(a,a) == 0; - assert stype2.compare(z,z) == 0; + assert baseType.compare(a,z) < 0; + assert baseType.compare(z,a) > 0; + assert baseType.compare(a,a) == 0; + assert baseType.compare(z,z) == 0; } - @Test public void testSimpleType1() { - CellName a = stype1.makeCellName(ByteBufferUtil.bytes("a")); - CellName z = stype1.makeCellName(ByteBufferUtil.bytes("z")); + CompositeType baseType = CompositeType.getInstance(BytesType.instance); + ByteBuffer a = baseType.builder().add(ByteBufferUtil.bytes("a")).build(); + ByteBuffer z = baseType.builder().add(ByteBufferUtil.bytes("z")).build(); - assert stype1.compare(a,z) < 0; - assert stype1.compare(z,a) > 0; - assert stype1.compare(a,a) == 0; - assert stype1.compare(z,z) == 0; + assert baseType.compare(a,z) < 0; + assert baseType.compare(z,a) > 0; + assert baseType.compare(a,a) == 0; + assert baseType.compare(z,z) == 0; } - } diff --git a/test/unit/org/apache/cassandra/db/filter/ColumnSliceTest.java b/test/unit/org/apache/cassandra/db/filter/ColumnSliceTest.java deleted file mode 100644 index 8ba2665216..0000000000 --- a/test/unit/org/apache/cassandra/db/filter/ColumnSliceTest.java +++ /dev/null @@ -1,495 +0,0 @@ -/* - * * Licensed to the Apache Software Foundation (ASF) under one - * * or more contributor license agreements. See the NOTICE file - * * distributed with this work for additional information - * * regarding copyright ownership. The ASF licenses this file - * * to you under the Apache License, Version 2.0 (the - * * "License"); you may not use this file except in compliance - * * with the License. You may obtain a copy of the License at - * * - * * http://www.apache.org/licenses/LICENSE-2.0 - * * - * * Unless required by applicable law or agreed to in writing, - * * software distributed under the License is distributed on an - * * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * * KIND, either express or implied. See the License for the - * * specific language governing permissions and limitations - * * under the License. - * */ -package org.apache.cassandra.db.filter; - - -import java.nio.ByteBuffer; -import java.util.*; - -import org.junit.Test; - -import org.apache.cassandra.db.composites.*; -import org.apache.cassandra.db.marshal.AbstractType; -import org.apache.cassandra.db.marshal.Int32Type; -import org.apache.cassandra.io.sstable.ColumnNameHelper; -import org.apache.cassandra.utils.ByteBufferUtil; - -import static org.junit.Assert.*; - -public class ColumnSliceTest -{ - private static final CellNameType simpleIntType = new SimpleDenseCellNameType(Int32Type.instance); - - @Test - public void testIntersectsSingleSlice() - { - List> types = new ArrayList<>(); - types.add(Int32Type.instance); - types.add(Int32Type.instance); - types.add(Int32Type.instance); - CompoundDenseCellNameType nameType = new CompoundDenseCellNameType(types); - - // filter falls entirely before sstable - ColumnSlice slice = new ColumnSlice(composite(0, 0, 0), composite(1, 0, 0)); - assertFalse(slice.intersects(columnNames(2, 0, 0), columnNames(3, 0, 0), nameType, false)); - - // same case, but with empty start - slice = new ColumnSlice(composite(), composite(1, 0, 0)); - assertFalse(slice.intersects(columnNames(2, 0, 0), columnNames(3, 0, 0), nameType, false)); - - // same case, but with missing components for start - slice = new ColumnSlice(composite(0), composite(1, 0, 0)); - assertFalse(slice.intersects(columnNames(2, 0, 0), columnNames(3, 0, 0), nameType, false)); - - // same case, but with missing components for start and end - slice = new ColumnSlice(composite(0), composite(1, 0)); - assertFalse(slice.intersects(columnNames(2, 0, 0), columnNames(3, 0, 0), nameType, false)); - - - // end of slice matches start of sstable for the first component, but not the second component - slice = new ColumnSlice(composite(0, 0, 0), composite(1, 0, 0)); - assertFalse(slice.intersects(columnNames(1, 1, 0), columnNames(3, 0, 0), nameType, false)); - - // same case, but with missing components for start - slice = new ColumnSlice(composite(0), composite(1, 0, 0)); - assertFalse(slice.intersects(columnNames(1, 1, 0), columnNames(3, 0, 0), nameType, false)); - - // same case, but with missing components for start and end - slice = new ColumnSlice(composite(0), composite(1, 0)); - assertFalse(slice.intersects(columnNames(1, 1, 0), columnNames(3, 0, 0), nameType, false)); - - // first two components match, but not the last - slice = new ColumnSlice(composite(0, 0, 0), composite(1, 1, 0)); - assertFalse(slice.intersects(columnNames(1, 1, 1), columnNames(3, 1, 1), nameType, false)); - - // all three components in slice end match the start of the sstable - slice = new ColumnSlice(composite(0, 0, 0), composite(1, 1, 1)); - assertTrue(slice.intersects(columnNames(1, 1, 1), columnNames(3, 1, 1), nameType, false)); - - - // filter falls entirely after sstable - slice = new ColumnSlice(composite(4, 0, 0), composite(4, 0, 0)); - assertFalse(slice.intersects(columnNames(2, 0, 0), columnNames(3, 0, 0), nameType, false)); - - // same case, but with empty end - slice = new ColumnSlice(composite(4, 0, 0), composite()); - assertFalse(slice.intersects(columnNames(2, 0, 0), columnNames(3, 0, 0), nameType, false)); - - // same case, but with missing components for end - slice = new ColumnSlice(composite(4, 0, 0), composite(1)); - assertFalse(slice.intersects(columnNames(2, 0, 0), columnNames(3, 0, 0), nameType, false)); - - // same case, but with missing components for start and end - slice = new ColumnSlice(composite(4, 0), composite(1)); - assertFalse(slice.intersects(columnNames(2, 0, 0), columnNames(3, 0, 0), nameType, false)); - - - // start of slice matches end of sstable for the first component, but not the second component - slice = new ColumnSlice(composite(1, 1, 1), composite(2, 0, 0)); - assertFalse(slice.intersects(columnNames(0, 0, 0), columnNames(1, 0, 0), nameType, false)); - - // start of slice matches end of sstable for the first two components, but not the last component - slice = new ColumnSlice(composite(1, 1, 1), composite(2, 0, 0)); - assertFalse(slice.intersects(columnNames(0, 0, 0), columnNames(1, 1, 0), nameType, false)); - - // all three components in the slice start match the end of the sstable - slice = new ColumnSlice(composite(1, 1, 1), composite(2, 0, 0)); - assertTrue(slice.intersects(columnNames(0, 0, 0), columnNames(1, 1, 1), nameType, false)); - - - // slice covers entire sstable (with no matching edges) - slice = new ColumnSlice(composite(0, 0, 0), composite(2, 0, 0)); - assertTrue(slice.intersects(columnNames(1, 0, 0), columnNames(1, 1, 1), nameType, false)); - - // same case, but with empty ends - slice = new ColumnSlice(composite(), composite()); - assertTrue(slice.intersects(columnNames(1, 0, 0), columnNames(1, 1, 1), nameType, false)); - - // same case, but with missing components - slice = new ColumnSlice(composite(0), composite(2, 0)); - assertTrue(slice.intersects(columnNames(1, 0, 0), columnNames(1, 1, 1), nameType, false)); - - // slice covers entire sstable (with matching start) - slice = new ColumnSlice(composite(1, 0, 0), composite(2, 0, 0)); - assertTrue(slice.intersects(columnNames(1, 0, 0), columnNames(1, 1, 1), nameType, false)); - - // slice covers entire sstable (with matching end) - slice = new ColumnSlice(composite(0, 0, 0), composite(1, 1, 1)); - assertTrue(slice.intersects(columnNames(1, 0, 0), columnNames(1, 1, 1), nameType, false)); - - // slice covers entire sstable (with matching start and end) - slice = new ColumnSlice(composite(1, 0, 0), composite(1, 1, 1)); - assertTrue(slice.intersects(columnNames(1, 0, 0), columnNames(1, 1, 1), nameType, false)); - - - // slice falls entirely within sstable (with matching start) - slice = new ColumnSlice(composite(1, 0, 0), composite(1, 1, 0)); - assertTrue(slice.intersects(columnNames(1, 0, 0), columnNames(1, 1, 1), nameType, false)); - - // same case, but with a missing end component - slice = new ColumnSlice(composite(1, 0, 0), composite(1, 1)); - assertTrue(slice.intersects(columnNames(1, 0, 0), columnNames(1, 1, 1), nameType, false)); - - // slice falls entirely within sstable (with matching end) - slice = new ColumnSlice(composite(1, 1, 0), composite(1, 1, 1)); - assertTrue(slice.intersects(columnNames(1, 0, 0), columnNames(1, 1, 1), nameType, false)); - - // same case, but with a missing start component - slice = new ColumnSlice(composite(1, 1), composite(1, 1, 1)); - assertTrue(slice.intersects(columnNames(1, 0, 0), columnNames(1, 1, 1), nameType, false)); - - - // slice falls entirely within sstable - slice = new ColumnSlice(composite(1, 1, 0), composite(1, 1, 1)); - assertTrue(slice.intersects(columnNames(1, 0, 0), columnNames(2, 2, 2), nameType, false)); - - // same case, but with a missing start component - slice = new ColumnSlice(composite(1, 1), composite(1, 1, 1)); - assertTrue(slice.intersects(columnNames(1, 0, 0), columnNames(2, 2, 2), nameType, false)); - - // same case, but with a missing start and end components - slice = new ColumnSlice(composite(1), composite(1, 2)); - assertTrue(slice.intersects(columnNames(1, 0, 0), columnNames(2, 2, 2), nameType, false)); - - // same case, but with an equal first component and missing start and end components - slice = new ColumnSlice(composite(1), composite(1)); - assertTrue(slice.intersects(columnNames(1, 0, 0), columnNames(2, 2, 2), nameType, false)); - - // slice falls entirely within sstable (slice start and end are the same) - slice = new ColumnSlice(composite(1, 1, 1), composite(1, 1, 1)); - assertTrue(slice.intersects(columnNames(1, 0, 0), columnNames(2, 2, 2), nameType, false)); - - - // slice starts within sstable, empty end - slice = new ColumnSlice(composite(1, 1, 1), composite()); - assertTrue(slice.intersects(columnNames(1, 0, 0), columnNames(2, 0, 0), nameType, false)); - - // same case, but with missing end components - slice = new ColumnSlice(composite(1, 1, 1), composite(3)); - assertTrue(slice.intersects(columnNames(1, 0, 0), columnNames(2, 0, 0), nameType, false)); - - // slice starts within sstable (matching sstable start), empty end - slice = new ColumnSlice(composite(1, 0, 0), composite()); - assertTrue(slice.intersects(columnNames(1, 0, 0), columnNames(2, 0, 0), nameType, false)); - - // same case, but with missing end components - slice = new ColumnSlice(composite(1, 0, 0), composite(3)); - assertTrue(slice.intersects(columnNames(1, 0, 0), columnNames(2, 0, 0), nameType, false)); - - // slice starts within sstable (matching sstable end), empty end - slice = new ColumnSlice(composite(2, 0, 0), composite()); - assertTrue(slice.intersects(columnNames(1, 0, 0), columnNames(2, 0, 0), nameType, false)); - - // same case, but with missing end components - slice = new ColumnSlice(composite(2, 0, 0), composite(3)); - assertTrue(slice.intersects(columnNames(1, 0, 0), columnNames(2, 0, 0), nameType, false)); - - - // slice ends within sstable, empty end - slice = new ColumnSlice(composite(), composite(1, 1, 1)); - assertTrue(slice.intersects(columnNames(1, 0, 0), columnNames(2, 0, 0), nameType, false)); - - // same case, but with missing start components - slice = new ColumnSlice(composite(0), composite(1, 1, 1)); - assertTrue(slice.intersects(columnNames(1, 0, 0), columnNames(2, 0, 0), nameType, false)); - - // slice ends within sstable (matching sstable start), empty start - slice = new ColumnSlice(composite(), composite(1, 0, 0)); - assertTrue(slice.intersects(columnNames(1, 0, 0), columnNames(2, 0, 0), nameType, false)); - - // same case, but with missing start components - slice = new ColumnSlice(composite(0), composite(1, 0, 0)); - assertTrue(slice.intersects(columnNames(1, 0, 0), columnNames(2, 0, 0), nameType, false)); - - // slice ends within sstable (matching sstable end), empty start - slice = new ColumnSlice(composite(), composite(2, 0, 0)); - assertTrue(slice.intersects(columnNames(1, 0, 0), columnNames(2, 0, 0), nameType, false)); - - // same case, but with missing start components - slice = new ColumnSlice(composite(0), composite(2, 0, 0)); - assertTrue(slice.intersects(columnNames(1, 0, 0), columnNames(2, 0, 0), nameType, false)); - - - // the slice technically falls within the sstable range, but since the first component is restricted to - // a single value, we can check that the second component does not fall within its min/max - slice = new ColumnSlice(composite(1, 2, 0), composite(1, 3, 0)); - assertFalse(slice.intersects(columnNames(1, 0, 0), columnNames(2, 1, 0), nameType, false)); - - // same case, but with a missing start component - slice = new ColumnSlice(composite(1, 2), composite(1, 3, 0)); - assertFalse(slice.intersects(columnNames(1, 0, 0), columnNames(2, 1, 0), nameType, false)); - - // same case, but with a missing end component - slice = new ColumnSlice(composite(1, 2, 0), composite(1, 3)); - assertFalse(slice.intersects(columnNames(1, 0, 0), columnNames(2, 1, 0), nameType, false)); - - // same case, but with a missing start and end components - slice = new ColumnSlice(composite(1, 2), composite(1, 3)); - assertFalse(slice.intersects(columnNames(1, 0, 0), columnNames(2, 1, 0), nameType, false)); - - // same case, but with missing start and end components and different lengths for start and end - slice = new ColumnSlice(composite(1, 2), composite(1)); - assertFalse(slice.intersects(columnNames(1, 0, 0), columnNames(2, 1, 0), nameType, false)); - - - // same as the previous set of tests, but the second component is equal in the slice start and end - slice = new ColumnSlice(composite(1, 2, 0), composite(1, 2, 0)); - assertFalse(slice.intersects(columnNames(1, 0, 0), columnNames(2, 1, 0), nameType, false)); - - // same case, but with a missing start component - slice = new ColumnSlice(composite(1, 2), composite(1, 2, 0)); - assertFalse(slice.intersects(columnNames(1, 0, 0), columnNames(2, 1, 0), nameType, false)); - - // same case, but with a missing end component - slice = new ColumnSlice(composite(1, 2, 0), composite(1, 2)); - assertFalse(slice.intersects(columnNames(1, 0, 0), columnNames(2, 1, 0), nameType, false)); - - // same case, but with a missing start and end components - slice = new ColumnSlice(composite(1, 2), composite(1, 2)); - assertFalse(slice.intersects(columnNames(1, 0, 0), columnNames(2, 1, 0), nameType, false)); - - // same as the previous tests, but it's the third component that doesn't fit in its range this time - slice = new ColumnSlice(composite(1, 1, 2), composite(1, 1, 3)); - assertFalse(slice.intersects(columnNames(1, 1, 0), columnNames(2, 2, 1), nameType, false)); - - // empty min/max column names - slice = new ColumnSlice(composite(), composite()); - assertTrue(slice.intersects(columnNames(), columnNames(), nameType, false)); - - slice = new ColumnSlice(composite(1), composite()); - assertTrue(slice.intersects(columnNames(), columnNames(), nameType, false)); - - slice = new ColumnSlice(composite(), composite(1)); - assertTrue(slice.intersects(columnNames(), columnNames(), nameType, false)); - - slice = new ColumnSlice(composite(1), composite(1)); - assertTrue(slice.intersects(columnNames(), columnNames(), nameType, false)); - - slice = new ColumnSlice(composite(), composite()); - assertTrue(slice.intersects(columnNames(), columnNames(1), nameType, false)); - - slice = new ColumnSlice(composite(), composite(1)); - assertTrue(slice.intersects(columnNames(), columnNames(1), nameType, false)); - - slice = new ColumnSlice(composite(), composite(1)); - assertTrue(slice.intersects(columnNames(), columnNames(2), nameType, false)); - - slice = new ColumnSlice(composite(), composite(2)); - assertTrue(slice.intersects(columnNames(), columnNames(1), nameType, false)); - - slice = new ColumnSlice(composite(2), composite(3)); - assertFalse(slice.intersects(columnNames(), columnNames(1), nameType, false)); - - // basic check on reversed slices - slice = new ColumnSlice(composite(1, 0, 0), composite(0, 0, 0)); - assertFalse(slice.intersects(columnNames(2, 0, 0), columnNames(3, 0, 0), nameType, true)); - - slice = new ColumnSlice(composite(1, 0, 0), composite(0, 0, 0)); - assertFalse(slice.intersects(columnNames(1, 1, 0), columnNames(3, 0, 0), nameType, true)); - - slice = new ColumnSlice(composite(1, 1, 1), composite(1, 1, 0)); - assertTrue(slice.intersects(columnNames(1, 0, 0), columnNames(2, 2, 2), nameType, true)); - } - - @Test - public void testDifferentMinMaxLengths() - { - List> types = new ArrayList<>(); - types.add(Int32Type.instance); - types.add(Int32Type.instance); - types.add(Int32Type.instance); - CompoundDenseCellNameType nameType = new CompoundDenseCellNameType(types); - - // slice does intersect - ColumnSlice slice = new ColumnSlice(composite(), composite()); - assertTrue(slice.intersects(columnNames(), columnNames(1), nameType, false)); - - slice = new ColumnSlice(composite(), composite()); - assertTrue(slice.intersects(columnNames(1), columnNames(1, 2), nameType, false)); - - slice = new ColumnSlice(composite(), composite(1)); - assertTrue(slice.intersects(columnNames(), columnNames(1), nameType, false)); - - slice = new ColumnSlice(composite(1), composite()); - assertTrue(slice.intersects(columnNames(), columnNames(1), nameType, false)); - - slice = new ColumnSlice(composite(1), composite(1)); - assertTrue(slice.intersects(columnNames(), columnNames(1), nameType, false)); - - slice = new ColumnSlice(composite(0), composite(1, 2, 3)); - assertTrue(slice.intersects(columnNames(), columnNames(1), nameType, false)); - - slice = new ColumnSlice(composite(1, 2, 3), composite(2)); - assertTrue(slice.intersects(columnNames(), columnNames(1), nameType, false)); - - // slice does not intersect - slice = new ColumnSlice(composite(2), composite(3, 4, 5)); - assertFalse(slice.intersects(columnNames(), columnNames(1), nameType, false)); - - slice = new ColumnSlice(composite(0), composite(0, 1, 2)); - assertFalse(slice.intersects(columnNames(1), columnNames(1, 2), nameType, false)); - } - @Test - public void testColumnNameHelper() - { - List> types = new ArrayList<>(); - types.add(Int32Type.instance); - types.add(Int32Type.instance); - types.add(Int32Type.instance); - CompoundDenseCellNameType nameType = new CompoundDenseCellNameType(types); - assertTrue(ColumnNameHelper.overlaps(columnNames(0, 0, 0), columnNames(3, 3, 3), columnNames(1, 1, 1), columnNames(2, 2, 2), nameType)); - assertFalse(ColumnNameHelper.overlaps(columnNames(0, 0, 0), columnNames(3, 3, 3), columnNames(4, 4, 4), columnNames(5, 5, 5), nameType)); - assertFalse(ColumnNameHelper.overlaps(columnNames(0, 0, 0), columnNames(3, 3, 3), columnNames(3, 3, 4), columnNames(5, 5, 5), nameType)); - assertTrue(ColumnNameHelper.overlaps(columnNames(0), columnNames(3, 3, 3), columnNames(1, 1), columnNames(5), nameType)); - } - - @Test - public void testDeoverlapSlices() - { - ColumnSlice[] slices; - ColumnSlice[] deoverlapped; - - // Preserve correct slices - slices = slices(s(0, 3), s(4, 5), s(6, 9)); - assertSlicesValid(slices); - assertSlicesEquals(slices, deoverlapSlices(slices)); - - // Simple overlap - slices = slices(s(0, 3), s(2, 5), s(8, 9)); - assertSlicesInvalid(slices); - assertSlicesEquals(slices(s(0, 5), s(8, 9)), deoverlapSlices(slices)); - - // Slice overlaps others fully - slices = slices(s(0, 10), s(2, 5), s(8, 9)); - assertSlicesInvalid(slices); - assertSlicesEquals(slices(s(0, 10)), deoverlapSlices(slices)); - - // Slice with empty end overlaps others fully - slices = slices(s(0, -1), s(2, 5), s(8, 9)); - assertSlicesInvalid(slices); - assertSlicesEquals(slices(s(0, -1)), deoverlapSlices(slices)); - - // Overlap with slices selecting only one element - slices = slices(s(0, 4), s(4, 4), s(4, 8)); - assertSlicesInvalid(slices); - assertSlicesEquals(slices(s(0, 8)), deoverlapSlices(slices)); - - // Unordered slices (without overlap) - slices = slices(s(4, 8), s(0, 3), s(9, 9)); - assertSlicesInvalid(slices); - assertSlicesEquals(slices(s(0, 3), s(4, 8), s(9, 9)), deoverlapSlices(slices)); - - // All range select but not by a single slice - slices = slices(s(5, -1), s(2, 5), s(-1, 2)); - assertSlicesInvalid(slices); - assertSlicesEquals(slices(s(-1, -1)), deoverlapSlices(slices)); - } - - @Test - public void testValidateSlices() - { - assertSlicesValid(slices(s(0, 3))); - assertSlicesValid(slices(s(3, 3))); - assertSlicesValid(slices(s(3, 3), s(4, 4))); - assertSlicesValid(slices(s(0, 3), s(4, 5), s(6, 9))); - assertSlicesValid(slices(s(-1, -1))); - assertSlicesValid(slices(s(-1, 3), s(4, -1))); - - assertSlicesInvalid(slices(s(3, 0))); - assertSlicesInvalid(slices(s(0, 2), s(2, 4))); - assertSlicesInvalid(slices(s(0, 2), s(1, 4))); - assertSlicesInvalid(slices(s(0, 2), s(3, 4), s(3, 4))); - assertSlicesInvalid(slices(s(-1, 2), s(3, -1), s(5, 9))); - } - - private static Composite composite(Integer ... components) - { - List> types = new ArrayList<>(); - types.add(Int32Type.instance); - types.add(Int32Type.instance); - types.add(Int32Type.instance); - CompoundDenseCellNameType nameType = new CompoundDenseCellNameType(types); - return nameType.make((Object[]) components); - } - - private static List columnNames(Integer ... components) - { - List names = new ArrayList<>(components.length); - for (int component : components) - names.add(ByteBufferUtil.bytes(component)); - return names; - } - - private static Composite simpleComposite(int i) - { - // We special negative values to mean EMPTY for convenience sake - if (i < 0) - return Composites.EMPTY; - - return simpleIntType.make(i); - } - - private static ColumnSlice s(int start, int finish) - { - return new ColumnSlice(simpleComposite(start), simpleComposite(finish)); - } - - private static ColumnSlice[] slices(ColumnSlice... slices) - { - return slices; - } - - private static ColumnSlice[] deoverlapSlices(ColumnSlice[] slices) - { - return ColumnSlice.deoverlapSlices(slices, simpleIntType); - } - - private static void assertSlicesValid(ColumnSlice[] slices) - { - assertTrue("Slices " + toString(slices) + " should be valid", ColumnSlice.validateSlices(slices, simpleIntType, false)); - } - - private static void assertSlicesInvalid(ColumnSlice[] slices) - { - assertFalse("Slices " + toString(slices) + " shouldn't be valid", ColumnSlice.validateSlices(slices, simpleIntType, false)); - } - - private static void assertSlicesEquals(ColumnSlice[] expected, ColumnSlice[] actual) - { - assertTrue("Expected " + toString(expected) + " but got " + toString(actual), Arrays.equals(expected, actual)); - } - - private static String toString(ColumnSlice[] slices) - { - StringBuilder sb = new StringBuilder().append("["); - for (int i = 0; i < slices.length; i++) - { - if (i > 0) - sb.append(", "); - - ColumnSlice slice = slices[i]; - sb.append("("); - sb.append(slice.start.isEmpty() ? "-1" : simpleIntType.getString(slice.start)); - sb.append(", "); - sb.append(slice.finish.isEmpty() ? "-1" : simpleIntType.getString(slice.finish)); - sb.append(")"); - } - return sb.append("]").toString(); - } -} diff --git a/test/unit/org/apache/cassandra/db/filter/SliceTest.java b/test/unit/org/apache/cassandra/db/filter/SliceTest.java new file mode 100644 index 0000000000..2f07a240c3 --- /dev/null +++ b/test/unit/org/apache/cassandra/db/filter/SliceTest.java @@ -0,0 +1,409 @@ +/* + * * Licensed to the Apache Software Foundation (ASF) under one + * * or more contributor license agreements. See the NOTICE file + * * distributed with this work for additional information + * * regarding copyright ownership. The ASF licenses this file + * * to you under the Apache License, Version 2.0 (the + * * "License"); you may not use this file except in compliance + * * with the License. You may obtain a copy of the License at + * * + * * http://www.apache.org/licenses/LICENSE-2.0 + * * + * * Unless required by applicable law or agreed to in writing, + * * software distributed under the License is distributed on an + * * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * * KIND, either express or implied. See the License for the + * * specific language governing permissions and limitations + * * under the License. + * */ +package org.apache.cassandra.db.filter; + + +import java.awt.*; +import java.nio.ByteBuffer; +import java.util.*; +import java.util.List; + +import org.apache.cassandra.config.ColumnDefinition; +import org.apache.cassandra.db.*; +import org.junit.Test; + +import org.apache.cassandra.db.marshal.AbstractType; +import org.apache.cassandra.db.marshal.Int32Type; +import org.apache.cassandra.utils.ByteBufferUtil; + +import static org.junit.Assert.*; + +public class SliceTest +{ + @Test + public void testIntersectsSingleSlice() + { + List> types = new ArrayList<>(); + types.add(Int32Type.instance); + types.add(Int32Type.instance); + types.add(Int32Type.instance); + ClusteringComparator cc = new ClusteringComparator(types); + + ClusteringPrefix.Kind sk = ClusteringPrefix.Kind.INCL_START_BOUND; + ClusteringPrefix.Kind ek = ClusteringPrefix.Kind.INCL_END_BOUND; + + // filter falls entirely before sstable + Slice slice = Slice.make(makeBound(sk, 0, 0, 0), makeBound(ek, 1, 0, 0)); + assertFalse(slice.intersects(cc, columnNames(2, 0, 0), columnNames(3, 0, 0))); + + // same case, but with empty start + slice = Slice.make(makeBound(sk), makeBound(ek, 1, 0, 0)); + assertFalse(slice.intersects(cc, columnNames(2, 0, 0), columnNames(3, 0, 0))); + + // same case, but with missing components for start + slice = Slice.make(makeBound(sk, 0), makeBound(ek, 1, 0, 0)); + assertFalse(slice.intersects(cc, columnNames(2, 0, 0), columnNames(3, 0, 0))); + + // same case, but with missing components for start and end + slice = Slice.make(makeBound(sk, 0), makeBound(ek, 1, 0)); + assertFalse(slice.intersects(cc, columnNames(2, 0, 0), columnNames(3, 0, 0))); + + + // end of slice matches start of sstable for the first component, but not the second component + slice = Slice.make(makeBound(sk, 0, 0, 0), makeBound(ek, 1, 0, 0)); + assertFalse(slice.intersects(cc, columnNames(1, 1, 0), columnNames(3, 0, 0))); + + // same case, but with missing components for start + slice = Slice.make(makeBound(sk, 0), makeBound(ek, 1, 0, 0)); + assertFalse(slice.intersects(cc, columnNames(1, 1, 0), columnNames(3, 0, 0))); + + // same case, but with missing components for start and end + slice = Slice.make(makeBound(sk, 0), makeBound(ek, 1, 0)); + assertFalse(slice.intersects(cc, columnNames(1, 1, 0), columnNames(3, 0, 0))); + + // first two components match, but not the last + slice = Slice.make(makeBound(sk, 0, 0, 0), makeBound(ek, 1, 1, 0)); + assertFalse(slice.intersects(cc, columnNames(1, 1, 1), columnNames(3, 1, 1))); + + // all three components in slice end match the start of the sstable + slice = Slice.make(makeBound(sk, 0, 0, 0), makeBound(ek, 1, 1, 1)); + assertTrue(slice.intersects(cc, columnNames(1, 1, 1), columnNames(3, 1, 1))); + + + // filter falls entirely after sstable + slice = Slice.make(makeBound(sk, 4, 0, 0), makeBound(ek, 4, 0, 0)); + assertFalse(slice.intersects(cc, columnNames(2, 0, 0), columnNames(3, 0, 0))); + + // same case, but with empty end + slice = Slice.make(makeBound(sk, 4, 0, 0), makeBound(ek)); + assertFalse(slice.intersects(cc, columnNames(2, 0, 0), columnNames(3, 0, 0))); + + // same case, but with missing components for end + slice = Slice.make(makeBound(sk, 4, 0, 0), makeBound(ek, 1)); + assertFalse(slice.intersects(cc, columnNames(2, 0, 0), columnNames(3, 0, 0))); + + // same case, but with missing components for start and end + slice = Slice.make(makeBound(sk, 4, 0), makeBound(ek, 1)); + assertFalse(slice.intersects(cc, columnNames(2, 0, 0), columnNames(3, 0, 0))); + + + // start of slice matches end of sstable for the first component, but not the second component + slice = Slice.make(makeBound(sk, 1, 1, 1), makeBound(ek, 2, 0, 0)); + assertFalse(slice.intersects(cc, columnNames(0, 0, 0), columnNames(1, 0, 0))); + + // start of slice matches end of sstable for the first two components, but not the last component + slice = Slice.make(makeBound(sk, 1, 1, 1), makeBound(ek, 2, 0, 0)); + assertFalse(slice.intersects(cc, columnNames(0, 0, 0), columnNames(1, 1, 0))); + + // all three components in the slice start match the end of the sstable + slice = Slice.make(makeBound(sk, 1, 1, 1), makeBound(ek, 2, 0, 0)); + assertTrue(slice.intersects(cc, columnNames(0, 0, 0), columnNames(1, 1, 1))); + + + // slice covers entire sstable (with no matching edges) + slice = Slice.make(makeBound(sk, 0, 0, 0), makeBound(ek, 2, 0, 0)); + assertTrue(slice.intersects(cc, columnNames(1, 0, 0), columnNames(1, 1, 1))); + + // same case, but with empty ends + slice = Slice.make(makeBound(sk), makeBound(ek)); + assertTrue(slice.intersects(cc, columnNames(1, 0, 0), columnNames(1, 1, 1))); + + // same case, but with missing components + slice = Slice.make(makeBound(sk, 0), makeBound(ek, 2, 0)); + assertTrue(slice.intersects(cc, columnNames(1, 0, 0), columnNames(1, 1, 1))); + + // slice covers entire sstable (with matching start) + slice = Slice.make(makeBound(sk, 1, 0, 0), makeBound(ek, 2, 0, 0)); + assertTrue(slice.intersects(cc, columnNames(1, 0, 0), columnNames(1, 1, 1))); + + // slice covers entire sstable (with matching end) + slice = Slice.make(makeBound(sk, 0, 0, 0), makeBound(ek, 1, 1, 1)); + assertTrue(slice.intersects(cc, columnNames(1, 0, 0), columnNames(1, 1, 1))); + + // slice covers entire sstable (with matching start and end) + slice = Slice.make(makeBound(sk, 1, 0, 0), makeBound(ek, 1, 1, 1)); + assertTrue(slice.intersects(cc, columnNames(1, 0, 0), columnNames(1, 1, 1))); + + + // slice falls entirely within sstable (with matching start) + slice = Slice.make(makeBound(sk, 1, 0, 0), makeBound(ek, 1, 1, 0)); + assertTrue(slice.intersects(cc, columnNames(1, 0, 0), columnNames(1, 1, 1))); + + // same case, but with a missing end component + slice = Slice.make(makeBound(sk, 1, 0, 0), makeBound(ek, 1, 1)); + assertTrue(slice.intersects(cc, columnNames(1, 0, 0), columnNames(1, 1, 1))); + + // slice falls entirely within sstable (with matching end) + slice = Slice.make(makeBound(sk, 1, 1, 0), makeBound(ek, 1, 1, 1)); + assertTrue(slice.intersects(cc, columnNames(1, 0, 0), columnNames(1, 1, 1))); + + // same case, but with a missing start component + slice = Slice.make(makeBound(sk, 1, 1), makeBound(ek, 1, 1, 1)); + assertTrue(slice.intersects(cc, columnNames(1, 0, 0), columnNames(1, 1, 1))); + + + // slice falls entirely within sstable + slice = Slice.make(makeBound(sk, 1, 1, 0), makeBound(ek, 1, 1, 1)); + assertTrue(slice.intersects(cc, columnNames(1, 0, 0), columnNames(2, 2, 2))); + + // same case, but with a missing start component + slice = Slice.make(makeBound(sk, 1, 1), makeBound(ek, 1, 1, 1)); + assertTrue(slice.intersects(cc, columnNames(1, 0, 0), columnNames(2, 2, 2))); + + // same case, but with a missing start and end components + slice = Slice.make(makeBound(sk, 1), makeBound(ek, 1, 2)); + assertTrue(slice.intersects(cc, columnNames(1, 0, 0), columnNames(2, 2, 2))); + + // same case, but with an equal first component and missing start and end components + slice = Slice.make(makeBound(sk, 1), makeBound(ek, 1)); + assertTrue(slice.intersects(cc, columnNames(1, 0, 0), columnNames(2, 2, 2))); + + // slice falls entirely within sstable (slice start and end are the same) + slice = Slice.make(makeBound(sk, 1, 1, 1), makeBound(ek, 1, 1, 1)); + assertTrue(slice.intersects(cc, columnNames(1, 0, 0), columnNames(2, 2, 2))); + + + // slice starts within sstable, empty end + slice = Slice.make(makeBound(sk, 1, 1, 1), makeBound(ek)); + assertTrue(slice.intersects(cc, columnNames(1, 0, 0), columnNames(2, 0, 0))); + + // same case, but with missing end components + slice = Slice.make(makeBound(sk, 1, 1, 1), makeBound(ek, 3)); + assertTrue(slice.intersects(cc, columnNames(1, 0, 0), columnNames(2, 0, 0))); + + // slice starts within sstable (matching sstable start), empty end + slice = Slice.make(makeBound(sk, 1, 0, 0), makeBound(ek)); + assertTrue(slice.intersects(cc, columnNames(1, 0, 0), columnNames(2, 0, 0))); + + // same case, but with missing end components + slice = Slice.make(makeBound(sk, 1, 0, 0), makeBound(ek, 3)); + assertTrue(slice.intersects(cc, columnNames(1, 0, 0), columnNames(2, 0, 0))); + + // slice starts within sstable (matching sstable end), empty end + slice = Slice.make(makeBound(sk, 2, 0, 0), makeBound(ek)); + assertTrue(slice.intersects(cc, columnNames(1, 0, 0), columnNames(2, 0, 0))); + + // same case, but with missing end components + slice = Slice.make(makeBound(sk, 2, 0, 0), makeBound(ek, 3)); + assertTrue(slice.intersects(cc, columnNames(1, 0, 0), columnNames(2, 0, 0))); + + + // slice ends within sstable, empty end + slice = Slice.make(makeBound(sk), makeBound(ek, 1, 1, 1)); + assertTrue(slice.intersects(cc, columnNames(1, 0, 0), columnNames(2, 0, 0))); + + // same case, but with missing start components + slice = Slice.make(makeBound(sk, 0), makeBound(ek, 1, 1, 1)); + assertTrue(slice.intersects(cc, columnNames(1, 0, 0), columnNames(2, 0, 0))); + + // slice ends within sstable (matching sstable start), empty start + slice = Slice.make(makeBound(sk), makeBound(ek, 1, 0, 0)); + assertTrue(slice.intersects(cc, columnNames(1, 0, 0), columnNames(2, 0, 0))); + + // same case, but with missing start components + slice = Slice.make(makeBound(sk, 0), makeBound(ek, 1, 0, 0)); + assertTrue(slice.intersects(cc, columnNames(1, 0, 0), columnNames(2, 0, 0))); + + // slice ends within sstable (matching sstable end), empty start + slice = Slice.make(makeBound(sk), makeBound(ek, 2, 0, 0)); + assertTrue(slice.intersects(cc, columnNames(1, 0, 0), columnNames(2, 0, 0))); + + // same case, but with missing start components + slice = Slice.make(makeBound(sk, 0), makeBound(ek, 2, 0, 0)); + assertTrue(slice.intersects(cc, columnNames(1, 0, 0), columnNames(2, 0, 0))); + + // the slice technically falls within the sstable range, but since the first component is restricted to + // a single value, we can check that the second component does not fall within its min/max + slice = Slice.make(makeBound(sk, 1, 2, 0), makeBound(ek, 1, 3, 0)); + assertFalse(slice.intersects(cc, columnNames(1, 0, 0), columnNames(2, 1, 0))); + + // same case, but with a missing start component + slice = Slice.make(makeBound(sk, 1, 2), makeBound(ek, 1, 3, 0)); + assertFalse(slice.intersects(cc, columnNames(1, 0, 0), columnNames(2, 1, 0))); + + // same case, but with a missing end component + slice = Slice.make(makeBound(sk, 1, 2, 0), makeBound(ek, 1, 3)); + assertFalse(slice.intersects(cc, columnNames(1, 0, 0), columnNames(2, 1, 0))); + + // same case, but with a missing start and end components + slice = Slice.make(makeBound(sk, 1, 2), makeBound(ek, 1, 3)); + assertFalse(slice.intersects(cc, columnNames(1, 0, 0), columnNames(2, 1, 0))); + + // same case, but with missing start and end components and different lengths for start and end + slice = Slice.make(makeBound(sk, 1, 2), makeBound(ek, 1)); + assertFalse(slice.intersects(cc, columnNames(1, 0, 0), columnNames(2, 1, 0))); + + + // same as the previous set of tests, but the second component is equal in the slice start and end + slice = Slice.make(makeBound(sk, 1, 2, 0), makeBound(ek, 1, 2, 0)); + assertFalse(slice.intersects(cc, columnNames(1, 0, 0), columnNames(2, 1, 0))); + + // same case, but with a missing start component + slice = Slice.make(makeBound(sk, 1, 2), makeBound(ek, 1, 2, 0)); + assertFalse(slice.intersects(cc, columnNames(1, 0, 0), columnNames(2, 1, 0))); + + // same case, but with a missing end component + slice = Slice.make(makeBound(sk, 1, 2, 0), makeBound(ek, 1, 2)); + assertFalse(slice.intersects(cc, columnNames(1, 0, 0), columnNames(2, 1, 0))); + + // same case, but with a missing start and end components + slice = Slice.make(makeBound(sk, 1, 2), makeBound(ek, 1, 2)); + assertFalse(slice.intersects(cc, columnNames(1, 0, 0), columnNames(2, 1, 0))); + + // same as the previous tests, but it's the third component that doesn't fit in its range this time + slice = Slice.make(makeBound(sk, 1, 1, 2), makeBound(ek, 1, 1, 3)); + assertFalse(slice.intersects(cc, columnNames(1, 1, 0), columnNames(2, 2, 1))); + + // empty min/max column names + slice = Slice.make(makeBound(sk), makeBound(ek)); + assertTrue(slice.intersects(cc, columnNames(), columnNames())); + + slice = Slice.make(makeBound(sk, 1), makeBound(ek)); + assertTrue(slice.intersects(cc, columnNames(), columnNames())); + + slice = Slice.make(makeBound(sk), makeBound(ek, 1)); + assertTrue(slice.intersects(cc, columnNames(), columnNames())); + + slice = Slice.make(makeBound(sk, 1), makeBound(ek, 1)); + assertTrue(slice.intersects(cc, columnNames(), columnNames())); + + slice = Slice.make(makeBound(sk), makeBound(ek)); + assertTrue(slice.intersects(cc, columnNames(), columnNames(1))); + + slice = Slice.make(makeBound(sk), makeBound(ek, 1)); + assertTrue(slice.intersects(cc, columnNames(), columnNames(1))); + + slice = Slice.make(makeBound(sk), makeBound(ek, 1)); + assertTrue(slice.intersects(cc, columnNames(), columnNames(2))); + + slice = Slice.make(makeBound(sk), makeBound(ek, 2)); + assertTrue(slice.intersects(cc, columnNames(), columnNames(1))); + + slice = Slice.make(makeBound(sk, 2), makeBound(ek, 3)); + assertFalse(slice.intersects(cc, columnNames(), columnNames(1))); + + // basic check on reversed slices + slice = Slice.make(makeBound(sk, 1, 0, 0), makeBound(ek, 0, 0, 0)); + assertFalse(slice.intersects(cc, columnNames(2, 0, 0), columnNames(3, 0, 0))); + + slice = Slice.make(makeBound(sk, 1, 0, 0), makeBound(ek, 0, 0, 0)); + assertFalse(slice.intersects(cc, columnNames(1, 1, 0), columnNames(3, 0, 0))); + + slice = Slice.make(makeBound(sk, 1, 1, 1), makeBound(ek, 1, 1, 0)); + assertTrue(slice.intersects(cc, columnNames(1, 0, 0), columnNames(2, 2, 2))); + } + + @Test + public void testDifferentMinMaxLengths() + { + List> types = new ArrayList<>(); + types.add(Int32Type.instance); + types.add(Int32Type.instance); + types.add(Int32Type.instance); + ClusteringComparator cc = new ClusteringComparator(types); + + ClusteringPrefix.Kind sk = ClusteringPrefix.Kind.INCL_START_BOUND; + ClusteringPrefix.Kind ek = ClusteringPrefix.Kind.INCL_END_BOUND; + + // slice does intersect + Slice slice = Slice.make(makeBound(sk), makeBound(ek)); + assertTrue(slice.intersects(cc, columnNames(), columnNames(1))); + + slice = Slice.make(makeBound(sk), makeBound(ek)); + assertTrue(slice.intersects(cc, columnNames(1), columnNames(1, 2))); + + slice = Slice.make(makeBound(sk), makeBound(ek, 1)); + assertTrue(slice.intersects(cc, columnNames(), columnNames(1))); + + slice = Slice.make(makeBound(sk, 1), makeBound(ek)); + assertTrue(slice.intersects(cc, columnNames(), columnNames(1))); + + slice = Slice.make(makeBound(sk, 1), makeBound(ek, 1)); + assertTrue(slice.intersects(cc, columnNames(), columnNames(1))); + + slice = Slice.make(makeBound(sk, 0), makeBound(ek, 1, 2, 3)); + assertTrue(slice.intersects(cc, columnNames(), columnNames(1))); + + slice = Slice.make(makeBound(sk, 1, 2, 3), makeBound(ek, 2)); + assertTrue(slice.intersects(cc, columnNames(), columnNames(1))); + + // slice does not intersect + slice = Slice.make(makeBound(sk, 2), makeBound(ek, 3, 4, 5)); + assertFalse(slice.intersects(cc, columnNames(), columnNames(1))); + + slice = Slice.make(makeBound(sk, 0), makeBound(ek, 0, 1, 2)); + assertFalse(slice.intersects(cc, columnNames(1), columnNames(1, 2))); + } + + @Test + public void testSliceNormalization() + { + List> types = new ArrayList<>(); + types.add(Int32Type.instance); + types.add(Int32Type.instance); + types.add(Int32Type.instance); + ClusteringComparator cc = new ClusteringComparator(types); + + assertSlicesNormalization(cc, slices(s(0, 2), s(2, 4)), slices(s(0, 4))); + assertSlicesNormalization(cc, slices(s(0, 2), s(1, 4)), slices(s(0, 4))); + assertSlicesNormalization(cc, slices(s(0, 2), s(3, 4), s(3, 4)), slices(s(0, 2), s(3, 4))); + assertSlicesNormalization(cc, slices(s(-1, 3), s(-1, 4)), slices(s(-1, 4))); + assertSlicesNormalization(cc, slices(s(-1, 2), s(-1, 3), s(5, 9)), slices(s(-1, 3), s(5, 9))); + } + + private static Slice.Bound makeBound(ClusteringPrefix.Kind kind, Integer... components) + { + ByteBuffer[] values = new ByteBuffer[components.length]; + for (int i = 0; i < components.length; i++) + { + values[i] = ByteBufferUtil.bytes(components[i]); + } + return Slice.Bound.create(kind, values); + } + + private static List columnNames(Integer ... components) + { + List names = new ArrayList<>(components.length); + for (int component : components) + names.add(ByteBufferUtil.bytes(component)); + return names; + } + + private static Slice s(int start, int finish) + { + return Slice.make(makeBound(ClusteringPrefix.Kind.INCL_START_BOUND, start), + makeBound(ClusteringPrefix.Kind.INCL_END_BOUND, finish)); + } + + private Slice[] slices(Slice... slices) + { + return slices; + } + + private static void assertSlicesNormalization(ClusteringComparator cc, Slice[] original, Slice[] expected) + { + Slices.Builder builder = new Slices.Builder(cc); + for (Slice s : original) + builder.add(s); + Slices slices = builder.build(); + assertEquals(expected.length, slices.size()); + for (int i = 0; i < expected.length; i++) + assertEquals(expected[i], slices.get(i)); + } +} diff --git a/test/unit/org/apache/cassandra/db/index/PerRowSecondaryIndexTest.java b/test/unit/org/apache/cassandra/db/index/PerRowSecondaryIndexTest.java index 6e35d1a0a8..158460b9b9 100644 --- a/test/unit/org/apache/cassandra/db/index/PerRowSecondaryIndexTest.java +++ b/test/unit/org/apache/cassandra/db/index/PerRowSecondaryIndexTest.java @@ -20,41 +20,35 @@ package org.apache.cassandra.db.index; import java.nio.ByteBuffer; import java.util.Arrays; -import java.util.List; import java.util.Set; import org.junit.Before; import org.junit.BeforeClass; import org.junit.Test; +import junit.framework.Assert; import org.apache.cassandra.SchemaLoader; -import org.apache.cassandra.Util; +import org.apache.cassandra.config.CFMetaData; import org.apache.cassandra.config.ColumnDefinition; -import org.apache.cassandra.config.DatabaseDescriptor; import org.apache.cassandra.config.KSMetaData; +import org.apache.cassandra.config.Schema; +import org.apache.cassandra.cql3.ColumnIdentifier; import org.apache.cassandra.cql3.QueryProcessor; import org.apache.cassandra.cql3.UntypedResultSet; -import org.apache.cassandra.db.Cell; -import org.apache.cassandra.db.ColumnFamily; -import org.apache.cassandra.db.ColumnFamilyStore; -import org.apache.cassandra.db.DecoratedKey; -import org.apache.cassandra.db.IndexExpression; -import org.apache.cassandra.db.Mutation; -import org.apache.cassandra.db.Row; -import org.apache.cassandra.db.composites.CellName; -import org.apache.cassandra.db.filter.ExtendedFilter; -import org.apache.cassandra.db.filter.QueryFilter; +import org.apache.cassandra.db.*; +import org.apache.cassandra.db.rows.*; +import org.apache.cassandra.db.filter.RowFilter; +import org.apache.cassandra.db.partitions.SingletonUnfilteredPartitionIterator; +import org.apache.cassandra.db.partitions.UnfilteredPartitionIterator; +import org.apache.cassandra.db.partitions.PartitionUpdate; import org.apache.cassandra.exceptions.ConfigurationException; import org.apache.cassandra.exceptions.InvalidRequestException; import org.apache.cassandra.locator.SimpleStrategy; import org.apache.cassandra.utils.ByteBufferUtil; +import org.apache.cassandra.utils.FBUtilities; import org.apache.cassandra.utils.concurrent.OpOrder; -import static org.junit.Assert.assertEquals; -import static org.junit.Assert.assertFalse; -import static org.junit.Assert.assertNotNull; -import static org.junit.Assert.assertTrue; -import static org.junit.Assert.fail; +import static org.junit.Assert.*; public class PerRowSecondaryIndexTest { @@ -73,9 +67,9 @@ public class PerRowSecondaryIndexTest { SchemaLoader.prepareServer(); SchemaLoader.createKeyspace(KEYSPACE1, - SimpleStrategy.class, - KSMetaData.optsWithRF(1), - SchemaLoader.perRowIndexedCFMD(KEYSPACE1, CF_INDEXED)); + SimpleStrategy.class, + KSMetaData.optsWithRF(1), + SchemaLoader.perRowIndexedCFMD(KEYSPACE1, CF_INDEXED)); } @Before @@ -87,24 +81,29 @@ public class PerRowSecondaryIndexTest @Test public void testIndexInsertAndUpdate() { - // create a row then test that the configured index instance was able to read the row - Mutation rm; - rm = new Mutation(KEYSPACE1, ByteBufferUtil.bytes("k1")); - rm.add("Indexed1", Util.cellname("indexed"), ByteBufferUtil.bytes("foo"), 1); - rm.applyUnsafe(); + int nowInSec = FBUtilities.nowInSeconds(); - ColumnFamily indexedRow = PerRowSecondaryIndexTest.TestIndex.LAST_INDEXED_ROW; + // create a row then test that the configured index instance was able to read the row + CFMetaData cfm = Schema.instance.getCFMetaData(KEYSPACE1, CF_INDEXED); + ColumnDefinition cdef = cfm.getColumnDefinition(new ColumnIdentifier("indexed", true)); + + RowUpdateBuilder builder = new RowUpdateBuilder(cfm, FBUtilities.timestampMicros(), "k1"); + builder.add("indexed", ByteBufferUtil.bytes("foo")); + builder.build().apply(); + + + UnfilteredRowIterator indexedRow = PerRowSecondaryIndexTest.TestIndex.LAST_INDEXED_PARTITION; assertNotNull(indexedRow); - assertEquals(ByteBufferUtil.bytes("foo"), indexedRow.getColumn(Util.cellname("indexed")).value()); + assertEquals(ByteBufferUtil.bytes("foo"), UnfilteredRowIterators.filter(indexedRow, nowInSec).next().getCell(cdef).value()); // update the row and verify what was indexed - rm = new Mutation(KEYSPACE1, ByteBufferUtil.bytes("k1")); - rm.add("Indexed1", Util.cellname("indexed"), ByteBufferUtil.bytes("bar"), 2); - rm.applyUnsafe(); + builder = new RowUpdateBuilder(cfm, FBUtilities.timestampMicros() + 1 , "k1"); + builder.add("indexed", ByteBufferUtil.bytes("bar")); + builder.build().apply(); - indexedRow = PerRowSecondaryIndexTest.TestIndex.LAST_INDEXED_ROW; + indexedRow = PerRowSecondaryIndexTest.TestIndex.LAST_INDEXED_PARTITION; assertNotNull(indexedRow); - assertEquals(ByteBufferUtil.bytes("bar"), indexedRow.getColumn(Util.cellname("indexed")).value()); + assertEquals(ByteBufferUtil.bytes("bar"), UnfilteredRowIterators.filter(indexedRow, nowInSec).next().getCell(cdef).value()); assertTrue(Arrays.equals("k1".getBytes(), PerRowSecondaryIndexTest.TestIndex.LAST_INDEXED_KEY.array())); } @@ -112,17 +111,19 @@ public class PerRowSecondaryIndexTest public void testColumnDelete() { // issue a column delete and test that the configured index instance was notified to update - Mutation rm; - rm = new Mutation(KEYSPACE1, ByteBufferUtil.bytes("k2")); - rm.delete("Indexed1", Util.cellname("indexed"), 1); - rm.applyUnsafe(); + CFMetaData cfm = Schema.instance.getCFMetaData(KEYSPACE1, CF_INDEXED); - ColumnFamily indexedRow = PerRowSecondaryIndexTest.TestIndex.LAST_INDEXED_ROW; + new RowUpdateBuilder(cfm, FBUtilities.timestampMicros(), "k2") + .noRowMarker() + .delete("indexed") + .build() + .apply(); + + UnfilteredRowIterator indexedRow = PerRowSecondaryIndexTest.TestIndex.LAST_INDEXED_PARTITION; assertNotNull(indexedRow); - for (Cell cell : indexedRow.getSortedColumns()) - assertFalse(cell.isLive()); - + //We filter tombstones now... + Assert.assertFalse(UnfilteredRowIterators.filter(indexedRow, FBUtilities.nowInSeconds()).hasNext()); assertTrue(Arrays.equals("k2".getBytes(), PerRowSecondaryIndexTest.TestIndex.LAST_INDEXED_KEY.array())); } @@ -130,26 +131,26 @@ public class PerRowSecondaryIndexTest public void testRowDelete() { // issue a row level delete and test that the configured index instance was notified to update - Mutation rm; - rm = new Mutation(KEYSPACE1, ByteBufferUtil.bytes("k3")); - rm.delete("Indexed1", 1); - rm.applyUnsafe(); + CFMetaData cfm = Schema.instance.getCFMetaData(KEYSPACE1, CF_INDEXED); + RowUpdateBuilder.deleteRow(cfm, FBUtilities.timestampMicros(), "k3").apply(); - ColumnFamily indexedRow = PerRowSecondaryIndexTest.TestIndex.LAST_INDEXED_ROW; + UnfilteredRowIterator indexedRow = PerRowSecondaryIndexTest.TestIndex.LAST_INDEXED_PARTITION; assertNotNull(indexedRow); - for (Cell cell : indexedRow.getSortedColumns()) - assertFalse(cell.isLive()); - + assertNotNull(indexedRow.partitionLevelDeletion()); + Assert.assertFalse(UnfilteredRowIterators.filter(indexedRow, FBUtilities.nowInSeconds()).hasNext()); assertTrue(Arrays.equals("k3".getBytes(), PerRowSecondaryIndexTest.TestIndex.LAST_INDEXED_KEY.array())); } - + @Test public void testInvalidSearch() { - Mutation rm; - rm = new Mutation(KEYSPACE1, ByteBufferUtil.bytes("k4")); - rm.add("Indexed1", Util.cellname("indexed"), ByteBufferUtil.bytes("foo"), 1); - rm.apply(); + + CFMetaData cfm = Schema.instance.getCFMetaData(KEYSPACE1, CF_INDEXED); + + RowUpdateBuilder builder = new RowUpdateBuilder(cfm, FBUtilities.timestampMicros(), "k1"); + builder.add("indexed", ByteBufferUtil.bytes("foo")); + builder.build().apply(); + // test we can search: UntypedResultSet result = QueryProcessor.executeInternal(String.format("SELECT * FROM \"%s\".\"Indexed1\" WHERE indexed = 'foo'", KEYSPACE1)); @@ -169,41 +170,30 @@ public class PerRowSecondaryIndexTest public static class TestIndex extends PerRowSecondaryIndex { - public static volatile boolean ACTIVE = true; - public static ColumnFamily LAST_INDEXED_ROW; + public static UnfilteredRowIterator LAST_INDEXED_PARTITION; public static ByteBuffer LAST_INDEXED_KEY; public static void reset() { - ACTIVE = true; LAST_INDEXED_KEY = null; - LAST_INDEXED_ROW = null; + LAST_INDEXED_PARTITION = null; } @Override - public boolean indexes(CellName name) + public void index(ByteBuffer rowKey, UnfilteredRowIterator cf) { - return ACTIVE; + LAST_INDEXED_PARTITION = cf; + LAST_INDEXED_KEY = rowKey; } - - @Override - public boolean indexes(ColumnDefinition cdef) + + public void index(ByteBuffer rowKey, PartitionUpdate atoms) { - return ACTIVE; - } - - @Override - public void index(ByteBuffer rowKey, ColumnFamily cf) - { - QueryFilter filter = QueryFilter.getIdentityFilter(DatabaseDescriptor.getPartitioner().decorateKey(rowKey), - baseCfs.getColumnFamilyName(), - System.currentTimeMillis()); - LAST_INDEXED_ROW = cf; + LAST_INDEXED_PARTITION = atoms.unfilteredIterator(); LAST_INDEXED_KEY = rowKey; } @Override - public void delete(DecoratedKey key, OpOrder.Group opGroup) + public void delete(ByteBuffer key, OpOrder.Group opGroup) { } @@ -225,28 +215,36 @@ public class PerRowSecondaryIndexTest @Override public String getIndexName() { - return this.getClass().getSimpleName(); + return null; } @Override - protected SecondaryIndexSearcher createSecondaryIndexSearcher(Set columns) + protected SecondaryIndexSearcher createSecondaryIndexSearcher(Set columns) { return new SecondaryIndexSearcher(baseCfs.indexManager, columns) { - @Override - public List search(ExtendedFilter filter) + public UnfilteredPartitionIterator search(ReadCommand filter, ReadOrderGroup orderGroup) { - return Arrays.asList(new Row(LAST_INDEXED_KEY, LAST_INDEXED_ROW)); + return new SingletonUnfilteredPartitionIterator(LAST_INDEXED_PARTITION, false); } @Override - public void validate(IndexExpression indexExpression) throws InvalidRequestException + public RowFilter.Expression primaryClause(ReadCommand command) { - if (indexExpression.value.equals(ByteBufferUtil.bytes("invalid"))) + RowFilter.Expression expression = command.rowFilter().iterator().next(); + + if (expression.getIndexValue().equals(ByteBufferUtil.bytes("invalid"))) throw new InvalidRequestException("Invalid search!"); + + return expression; + } + + protected UnfilteredPartitionIterator queryDataFromIndex(AbstractSimplePerColumnSecondaryIndex index, DecoratedKey indexKey, RowIterator indexHits, ReadCommand command, ReadOrderGroup orderGroup) + { + // As we override 'search()' directly for the test, we don't care about this. + throw new UnsupportedOperationException(); } - }; } @@ -261,6 +259,30 @@ public class PerRowSecondaryIndexTest return baseCfs; } + @Override + public boolean indexes(ColumnDefinition name) + { + return true; + } + + @Override + public void validate(DecoratedKey partitionKey) throws InvalidRequestException + { + + } + + @Override + public void validate(Clustering clustering) throws InvalidRequestException + { + + } + + @Override + public void validate(ByteBuffer cellValue, CellPath path) throws InvalidRequestException + { + + } + @Override public void removeIndex(ByteBuffer columnName) { diff --git a/test/unit/org/apache/cassandra/db/lifecycle/ViewTest.java b/test/unit/org/apache/cassandra/db/lifecycle/ViewTest.java index 4c8006a7c8..d4ecb18f12 100644 --- a/test/unit/org/apache/cassandra/db/lifecycle/ViewTest.java +++ b/test/unit/org/apache/cassandra/db/lifecycle/ViewTest.java @@ -33,7 +33,7 @@ import junit.framework.Assert; import org.apache.cassandra.MockSchema; import org.apache.cassandra.db.ColumnFamilyStore; import org.apache.cassandra.db.Memtable; -import org.apache.cassandra.db.RowPosition; +import org.apache.cassandra.db.PartitionPosition; import org.apache.cassandra.dht.AbstractBounds; import org.apache.cassandra.io.sstable.format.SSTableReader; @@ -59,8 +59,8 @@ public class ViewTest { for (int j = i ; j < 5 ; j++) { - RowPosition min = MockSchema.readerBounds(i); - RowPosition max = MockSchema.readerBounds(j); + PartitionPosition min = MockSchema.readerBounds(i); + PartitionPosition max = MockSchema.readerBounds(j); for (boolean minInc : new boolean[] { true, false} ) { for (boolean maxInc : new boolean[] { true, false} ) diff --git a/test/unit/org/apache/cassandra/db/marshal/CompositeTypeTest.java b/test/unit/org/apache/cassandra/db/marshal/CompositeTypeTest.java index 25c1c7de54..68b888c88d 100644 --- a/test/unit/org/apache/cassandra/db/marshal/CompositeTypeTest.java +++ b/test/unit/org/apache/cassandra/db/marshal/CompositeTypeTest.java @@ -19,6 +19,7 @@ package org.apache.cassandra.db.marshal; import java.nio.ByteBuffer; +import java.nio.charset.CharacterCodingException; import java.util.*; import org.junit.BeforeClass; @@ -28,15 +29,17 @@ import static org.junit.Assert.assertEquals; import org.apache.cassandra.SchemaLoader; import org.apache.cassandra.Util; -import org.apache.cassandra.config.CFMetaData; +import org.apache.cassandra.config.ColumnDefinition; import org.apache.cassandra.config.KSMetaData; +import org.apache.cassandra.db.*; +import org.apache.cassandra.db.rows.Cell; +import org.apache.cassandra.db.rows.Row; +import org.apache.cassandra.db.partitions.ArrayBackedPartition; import org.apache.cassandra.exceptions.ConfigurationException; import org.apache.cassandra.exceptions.SyntaxException; -import org.apache.cassandra.db.*; -import org.apache.cassandra.db.composites.CellNames; -import org.apache.cassandra.db.filter.QueryFilter; import org.apache.cassandra.locator.SimpleStrategy; import org.apache.cassandra.serializers.MarshalException; +import org.apache.cassandra.service.StorageService; import org.apache.cassandra.utils.*; public class CompositeTypeTest @@ -69,7 +72,7 @@ public class CompositeTypeTest SchemaLoader.createKeyspace(KEYSPACE1, SimpleStrategy.class, KSMetaData.optsWithRF(1), - SchemaLoader.standardCFMD(KEYSPACE1, CF_STANDARDCOMPOSITE, composite)); + SchemaLoader.denseCFMD(KEYSPACE1, CF_STANDARDCOMPOSITE, composite)); } @Test @@ -187,23 +190,28 @@ public class CompositeTypeTest ByteBuffer cname5 = createCompositeKey("test2", uuids[1], 42, false); ByteBuffer key = ByteBufferUtil.bytes("k"); - Mutation rm = new Mutation(KEYSPACE1, key); - addColumn(rm, cname5); - addColumn(rm, cname1); - addColumn(rm, cname4); - addColumn(rm, cname2); - addColumn(rm, cname3); - rm.applyUnsafe(); - ColumnFamily cf = cfs.getColumnFamily(QueryFilter.getIdentityFilter(Util.dk("k"), CF_STANDARDCOMPOSITE, System.currentTimeMillis())); + long ts = FBUtilities.timestampMicros(); + new RowUpdateBuilder(cfs.metadata, ts, key).clustering(cname5).add("val", "cname5").build().applyUnsafe(); + new RowUpdateBuilder(cfs.metadata, ts, key).clustering(cname1).add("val", "cname1").build().applyUnsafe(); + new RowUpdateBuilder(cfs.metadata, ts, key).clustering(cname4).add("val", "cname4").build().applyUnsafe(); + new RowUpdateBuilder(cfs.metadata, ts, key).clustering(cname2).add("val", "cname2").build().applyUnsafe(); + new RowUpdateBuilder(cfs.metadata, ts, key).clustering(cname3).add("val", "cname3").build().applyUnsafe(); - Iterator iter = cf.getSortedColumns().iterator(); + ColumnDefinition cdef = cfs.metadata.getColumnDefinition(ByteBufferUtil.bytes("val")); - assert iter.next().name().toByteBuffer().equals(cname1); - assert iter.next().name().toByteBuffer().equals(cname2); - assert iter.next().name().toByteBuffer().equals(cname3); - assert iter.next().name().toByteBuffer().equals(cname4); - assert iter.next().name().toByteBuffer().equals(cname5); + ArrayBackedPartition readPartition = Util.getOnlyPartitionUnfiltered(Util.cmd(cfs, key).build()); + Iterator iter = readPartition.iterator(); + + compareValues(iter.next().getCell(cdef), "cname1"); + compareValues(iter.next().getCell(cdef), "cname2"); + compareValues(iter.next().getCell(cdef), "cname3"); + compareValues(iter.next().getCell(cdef), "cname4"); + compareValues(iter.next().getCell(cdef), "cname5"); + } + private void compareValues(Cell c, String r) throws CharacterCodingException + { + assert ByteBufferUtil.string(c.value()).equals(r) : "Expected: {" + ByteBufferUtil.string(c.value()) + "} got: {" + r + "}"; } @Test @@ -214,14 +222,16 @@ public class CompositeTypeTest TypeParser.parse("CompositeType"); fail("Shouldn't work"); } - catch (ConfigurationException | SyntaxException e) {} + catch (ConfigurationException e) {} + catch (SyntaxException e) {} try { TypeParser.parse("CompositeType()"); fail("Shouldn't work"); } - catch (ConfigurationException | SyntaxException e) {} + catch (ConfigurationException e) {} + catch (SyntaxException e) {} } @Test @@ -267,11 +277,6 @@ public class CompositeTypeTest } } - private void addColumn(Mutation rm, ByteBuffer cname) - { - rm.add(CF_STANDARDCOMPOSITE, CellNames.simpleDense(cname), ByteBufferUtil.EMPTY_BYTE_BUFFER, 0); - } - private ByteBuffer createCompositeKey(String s, UUID uuid, int i, boolean lastIsOne) { ByteBuffer bytes = ByteBufferUtil.bytes(s); diff --git a/test/unit/org/apache/cassandra/db/marshal/DynamicCompositeTypeTest.java b/test/unit/org/apache/cassandra/db/marshal/DynamicCompositeTypeTest.java index 1a6ddc91ed..071c5de491 100644 --- a/test/unit/org/apache/cassandra/db/marshal/DynamicCompositeTypeTest.java +++ b/test/unit/org/apache/cassandra/db/marshal/DynamicCompositeTypeTest.java @@ -19,8 +19,9 @@ package org.apache.cassandra.db.marshal; import java.nio.ByteBuffer; -import java.util.Iterator; +import java.nio.charset.CharacterCodingException; import java.util.HashMap; +import java.util.Iterator; import java.util.Map; import java.util.UUID; @@ -30,14 +31,16 @@ import static org.junit.Assert.fail; import org.apache.cassandra.SchemaLoader; import org.apache.cassandra.Util; -import org.apache.cassandra.config.CFMetaData; +import org.apache.cassandra.config.ColumnDefinition; import org.apache.cassandra.config.KSMetaData; import org.apache.cassandra.db.*; -import org.apache.cassandra.db.composites.*; -import org.apache.cassandra.db.filter.QueryFilter; +import org.apache.cassandra.db.rows.Cell; +import org.apache.cassandra.db.rows.Row; +import org.apache.cassandra.db.partitions.ArrayBackedPartition; import org.apache.cassandra.exceptions.ConfigurationException; import org.apache.cassandra.locator.SimpleStrategy; import org.apache.cassandra.serializers.MarshalException; +import org.apache.cassandra.service.StorageService; import org.apache.cassandra.utils.*; public class DynamicCompositeTypeTest @@ -72,7 +75,7 @@ public class DynamicCompositeTypeTest SchemaLoader.createKeyspace(KEYSPACE1, SimpleStrategy.class, KSMetaData.optsWithRF(1), - SchemaLoader.standardCFMD(KEYSPACE1, CF_STANDARDDYNCOMPOSITE, dynamicComposite)); + SchemaLoader.denseCFMD(KEYSPACE1, CF_STANDARDDYNCOMPOSITE, dynamicComposite)); } @Test @@ -192,23 +195,27 @@ public class DynamicCompositeTypeTest ByteBuffer cname5 = createDynamicCompositeKey("test2", uuids[1], 42, false); ByteBuffer key = ByteBufferUtil.bytes("k"); - Mutation rm = new Mutation(KEYSPACE1, key); - addColumn(rm, cname5); - addColumn(rm, cname1); - addColumn(rm, cname4); - addColumn(rm, cname2); - addColumn(rm, cname3); - rm.applyUnsafe(); + long ts = FBUtilities.timestampMicros(); + new RowUpdateBuilder(cfs.metadata, ts, key).clustering(cname5).add("val", "cname5").build().applyUnsafe(); + new RowUpdateBuilder(cfs.metadata, ts, key).clustering(cname1).add("val", "cname1").build().applyUnsafe(); + new RowUpdateBuilder(cfs.metadata, ts, key).clustering(cname4).add("val", "cname4").build().applyUnsafe(); + new RowUpdateBuilder(cfs.metadata, ts, key).clustering(cname2).add("val", "cname2").build().applyUnsafe(); + new RowUpdateBuilder(cfs.metadata, ts, key).clustering(cname3).add("val", "cname3").build().applyUnsafe(); - ColumnFamily cf = cfs.getColumnFamily(QueryFilter.getIdentityFilter(Util.dk("k"), CF_STANDARDDYNCOMPOSITE, System.currentTimeMillis())); + ColumnDefinition cdef = cfs.metadata.getColumnDefinition(ByteBufferUtil.bytes("val")); - Iterator iter = cf.getSortedColumns().iterator(); + ArrayBackedPartition readPartition = Util.getOnlyPartitionUnfiltered(Util.cmd(cfs, key).build()); + Iterator iter = readPartition.iterator(); - assert iter.next().name().toByteBuffer().equals(cname1); - assert iter.next().name().toByteBuffer().equals(cname2); - assert iter.next().name().toByteBuffer().equals(cname3); - assert iter.next().name().toByteBuffer().equals(cname4); - assert iter.next().name().toByteBuffer().equals(cname5); + compareValues(iter.next().getCell(cdef), "cname1"); + compareValues(iter.next().getCell(cdef), "cname2"); + compareValues(iter.next().getCell(cdef), "cname3"); + compareValues(iter.next().getCell(cdef), "cname4"); + compareValues(iter.next().getCell(cdef), "cname5"); + } + private void compareValues(Cell c, String r) throws CharacterCodingException + { + assert ByteBufferUtil.string(c.value()).equals(r) : "Expected: {" + ByteBufferUtil.string(c.value()) + "} got: {" + r + "}"; } @Test @@ -224,23 +231,24 @@ public class DynamicCompositeTypeTest ByteBuffer cname5 = createDynamicCompositeKey("test2", uuids[1], 42, false, true); ByteBuffer key = ByteBufferUtil.bytes("kr"); - Mutation rm = new Mutation(KEYSPACE1, key); - addColumn(rm, cname5); - addColumn(rm, cname1); - addColumn(rm, cname4); - addColumn(rm, cname2); - addColumn(rm, cname3); - rm.apply(); - ColumnFamily cf = cfs.getColumnFamily(QueryFilter.getIdentityFilter(Util.dk("kr"), CF_STANDARDDYNCOMPOSITE, System.currentTimeMillis())); + long ts = FBUtilities.timestampMicros(); + new RowUpdateBuilder(cfs.metadata, ts, key).clustering(cname5).add("val", "cname5").build().applyUnsafe(); + new RowUpdateBuilder(cfs.metadata, ts, key).clustering(cname1).add("val", "cname1").build().applyUnsafe(); + new RowUpdateBuilder(cfs.metadata, ts, key).clustering(cname4).add("val", "cname4").build().applyUnsafe(); + new RowUpdateBuilder(cfs.metadata, ts, key).clustering(cname2).add("val", "cname2").build().applyUnsafe(); + new RowUpdateBuilder(cfs.metadata, ts, key).clustering(cname3).add("val", "cname3").build().applyUnsafe(); - Iterator iter = cf.getSortedColumns().iterator(); + ColumnDefinition cdef = cfs.metadata.getColumnDefinition(ByteBufferUtil.bytes("val")); - assert iter.next().name().toByteBuffer().equals(cname5); - assert iter.next().name().toByteBuffer().equals(cname4); - assert iter.next().name().toByteBuffer().equals(cname1); // null UUID < reversed value - assert iter.next().name().toByteBuffer().equals(cname3); - assert iter.next().name().toByteBuffer().equals(cname2); + ArrayBackedPartition readPartition = Util.getOnlyPartitionUnfiltered(Util.cmd(cfs, key).build()); + Iterator iter = readPartition.iterator(); + + compareValues(iter.next().getCell(cdef), "cname5"); + compareValues(iter.next().getCell(cdef), "cname4"); + compareValues(iter.next().getCell(cdef), "cname1"); // null UUID < reversed value + compareValues(iter.next().getCell(cdef), "cname3"); + compareValues(iter.next().getCell(cdef), "cname2"); } @Test @@ -309,11 +317,6 @@ public class DynamicCompositeTypeTest assert !TypeParser.parse("DynamicCompositeType(a => BytesType)").isCompatibleWith(TypeParser.parse("DynamicCompositeType(a => BytesType, b => AsciiType)")); } - private void addColumn(Mutation rm, ByteBuffer cname) - { - rm.add(CF_STANDARDDYNCOMPOSITE, CellNames.simpleDense(cname), ByteBufferUtil.EMPTY_BYTE_BUFFER, 0); - } - private ByteBuffer createDynamicCompositeKey(String s, UUID uuid, int i, boolean lastIsOne) { return createDynamicCompositeKey(s, uuid, i, lastIsOne, false); diff --git a/test/unit/org/apache/cassandra/db/marshal/TypeCompareTest.java b/test/unit/org/apache/cassandra/db/marshal/TypeCompareTest.java index fae04a2bac..04b030e13a 100644 --- a/test/unit/org/apache/cassandra/db/marshal/TypeCompareTest.java +++ b/test/unit/org/apache/cassandra/db/marshal/TypeCompareTest.java @@ -92,48 +92,6 @@ public class TypeCompareTest } } - @Test - public void testByte() - { - Random rng = new Random(); - ByteBuffer[] data = new ByteBuffer[Byte.MAX_VALUE]; - for (int i = 0; i < data.length; i++) - { - data[i] = ByteBuffer.allocate(1); - rng.nextBytes(data[i].array()); - } - - Arrays.sort(data, ByteType.instance); - - for (int i = 1; i < data.length; i++) - { - byte b0 = data[i - 1].get(data[i - 1].position()); - byte b1 = data[i].get(data[i].position()); - assert b0 <= b1; - } - } - - @Test - public void testShort() - { - Random rng = new Random(); - ByteBuffer[] data = new ByteBuffer[1000]; - for (int i = 0; i < data.length; i++) - { - data[i] = ByteBuffer.allocate(2); - rng.nextBytes(data[i].array()); - } - - Arrays.sort(data, ShortType.instance); - - for (int i = 1; i < data.length; i++) - { - short s0 = data[i - 1].getShort(data[i - 1].position()); - short s1 = data[i].getShort(data[i].position()); - assert s0 <= s1; - } - } - @Test public void testInt() { diff --git a/test/unit/org/apache/cassandra/db/marshal/TypeParserTest.java b/test/unit/org/apache/cassandra/db/marshal/TypeParserTest.java index 6581fc7bce..ee3052c17a 100644 --- a/test/unit/org/apache/cassandra/db/marshal/TypeParserTest.java +++ b/test/unit/org/apache/cassandra/db/marshal/TypeParserTest.java @@ -40,12 +40,6 @@ public class TypeParserTest type = TypeParser.parse(" "); assert type == BytesType.instance; - type = TypeParser.parse("ByteType"); - assert type == ByteType.instance; - - type = TypeParser.parse("ShortType"); - assert type == ShortType.instance; - type = TypeParser.parse("LongType"); assert type == LongType.instance; @@ -75,13 +69,15 @@ public class TypeParserTest TypeParser.parse("y"); fail("Should not pass"); } - catch (ConfigurationException | SyntaxException e) {} + catch (ConfigurationException e) {} + catch (SyntaxException e) {} try { TypeParser.parse("LongType(reversed@)"); fail("Should not pass"); } - catch (ConfigurationException | SyntaxException e) {} + catch (ConfigurationException e) {} + catch (SyntaxException e) {} } } diff --git a/test/unit/org/apache/cassandra/db/marshal/TypeValidationTest.java b/test/unit/org/apache/cassandra/db/marshal/TypeValidationTest.java index 5ebeb64b5d..ed5e2bf81c 100644 --- a/test/unit/org/apache/cassandra/db/marshal/TypeValidationTest.java +++ b/test/unit/org/apache/cassandra/db/marshal/TypeValidationTest.java @@ -64,32 +64,6 @@ public class TypeValidationTest Int32Type.instance.validate(Util.getBytes(2057022603)); } - @Test - public void testValidShort() - { - ShortType.instance.validate(Util.getBytes((short) 5)); - ShortType.instance.validate(Util.getBytes(Short.MAX_VALUE)); - } - - @Test(expected = MarshalException.class) - public void testInvalidShort() - { - ShortType.instance.validate(Util.getBytes(2057022603)); - } - - @Test - public void testValidByte() - { - ByteType.instance.validate(Util.getBytes((byte) 5)); - ByteType.instance.validate(Util.getBytes(Byte.MAX_VALUE)); - } - - @Test(expected = MarshalException.class) - public void testInvalidByte() - { - ByteType.instance.validate(Util.getBytes(2057022603)); - } - @Test public void testValidUtf8() throws UnsupportedEncodingException { diff --git a/test/unit/org/apache/cassandra/dht/BootStrapperTest.java b/test/unit/org/apache/cassandra/dht/BootStrapperTest.java index 7c7fb42978..318537643e 100644 --- a/test/unit/org/apache/cassandra/dht/BootStrapperTest.java +++ b/test/unit/org/apache/cassandra/dht/BootStrapperTest.java @@ -1,21 +1,20 @@ /* -* Licensed to the Apache Software Foundation (ASF) under one -* or more contributor license agreements. See the NOTICE file -* distributed with this work for additional information -* regarding copyright ownership. The ASF licenses this file -* to you under the Apache License, Version 2.0 (the -* "License"); you may not use this file except in compliance -* with the License. You may obtain a copy of the License at -* -* http://www.apache.org/licenses/LICENSE-2.0 -* -* Unless required by applicable law or agreed to in writing, -* software distributed under the License is distributed on an -* "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -* KIND, either express or implied. See the License for the -* specific language governing permissions and limitations -* under the License. -*/ + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ package org.apache.cassandra.dht; import static org.junit.Assert.assertEquals; diff --git a/test/unit/org/apache/cassandra/dht/ByteOrderedPartitionerTest.java b/test/unit/org/apache/cassandra/dht/ByteOrderedPartitionerTest.java index e70e086136..c4896a33be 100644 --- a/test/unit/org/apache/cassandra/dht/ByteOrderedPartitionerTest.java +++ b/test/unit/org/apache/cassandra/dht/ByteOrderedPartitionerTest.java @@ -1,21 +1,20 @@ /* -* Licensed to the Apache Software Foundation (ASF) under one -* or more contributor license agreements. See the NOTICE file -* distributed with this work for additional information -* regarding copyright ownership. The ASF licenses this file -* to you under the Apache License, Version 2.0 (the -* "License"); you may not use this file except in compliance -* with the License. You may obtain a copy of the License at -* -* http://www.apache.org/licenses/LICENSE-2.0 -* -* Unless required by applicable law or agreed to in writing, -* software distributed under the License is distributed on an -* "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -* KIND, either express or implied. See the License for the -* specific language governing permissions and limitations -* under the License. -*/ + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ package org.apache.cassandra.dht; public class ByteOrderedPartitionerTest extends PartitionerTestCase diff --git a/test/unit/org/apache/cassandra/dht/KeyCollisionTest.java b/test/unit/org/apache/cassandra/dht/KeyCollisionTest.java index e8a5ee2883..9ee60d6c76 100644 --- a/test/unit/org/apache/cassandra/dht/KeyCollisionTest.java +++ b/test/unit/org/apache/cassandra/dht/KeyCollisionTest.java @@ -1,4 +1,4 @@ -/** +/* * Licensed to the Apache Software Foundation (ASF) under one * or more contributor license agreements. See the NOTICE file * distributed with this work for additional information @@ -7,35 +7,48 @@ * "License"); you may not use this file except in compliance * with the License. You may obtain a copy of the License at * - * http://www.apache.org/licenses/LICENSE-2.0 + * http://www.apache.org/licenses/LICENSE-2.0 * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. */ package org.apache.cassandra.dht; import java.math.BigInteger; import java.nio.ByteBuffer; -import java.util.*; +import java.util.ArrayList; +import java.util.HashMap; +import java.util.List; +import java.util.Map; +import java.util.Random; import org.junit.AfterClass; import org.junit.BeforeClass; import org.junit.Test; + import org.apache.cassandra.SchemaLoader; import org.apache.cassandra.Util; -import org.apache.cassandra.db.*; -import org.apache.cassandra.db.columniterator.IdentityQueryFilter; +import org.apache.cassandra.config.CFMetaData; +import org.apache.cassandra.config.DatabaseDescriptor; +import org.apache.cassandra.config.KSMetaData; +import org.apache.cassandra.config.Schema; +import org.apache.cassandra.db.BufferDecoratedKey; +import org.apache.cassandra.db.ColumnFamilyStore; +import org.apache.cassandra.db.DecoratedKey; +import org.apache.cassandra.db.Keyspace; +import org.apache.cassandra.db.RowUpdateBuilder; import org.apache.cassandra.db.marshal.AbstractType; import org.apache.cassandra.db.marshal.IntegerType; -import org.apache.cassandra.config.*; +import org.apache.cassandra.db.partitions.*; import org.apache.cassandra.exceptions.ConfigurationException; import org.apache.cassandra.locator.SimpleStrategy; import org.apache.cassandra.service.StorageService; -import org.apache.cassandra.utils.*; +import org.apache.cassandra.utils.ByteBufferUtil; +import org.apache.cassandra.utils.FBUtilities; +import org.apache.cassandra.utils.Pair; import static org.apache.cassandra.Util.dk; @@ -82,12 +95,12 @@ public class KeyCollisionTest insert("key1", "key2", "key3"); // token = 4 insert("longKey1", "longKey2"); // token = 8 - List rows = cfs.getRangeSlice(new Bounds(dk("k2"), dk("key2")), null, new IdentityQueryFilter(), 10000); - assert rows.size() == 4 : "Expecting 4 keys, got " + rows.size(); - assert rows.get(0).key.getKey().equals(ByteBufferUtil.bytes("k2")); - assert rows.get(1).key.getKey().equals(ByteBufferUtil.bytes("kq")); - assert rows.get(2).key.getKey().equals(ByteBufferUtil.bytes("key1")); - assert rows.get(3).key.getKey().equals(ByteBufferUtil.bytes("key2")); + List partitions = Util.getAll(Util.cmd(cfs).fromKeyIncl("k2").toKeyIncl("key2").build()); + + assert partitions.get(0).partitionKey().getKey().equals(ByteBufferUtil.bytes("k2")); + assert partitions.get(1).partitionKey().getKey().equals(ByteBufferUtil.bytes("kq")); + assert partitions.get(2).partitionKey().getKey().equals(ByteBufferUtil.bytes("key1")); + assert partitions.get(3).partitionKey().getKey().equals(ByteBufferUtil.bytes("key2")); } private void insert(String... keys) @@ -98,10 +111,8 @@ public class KeyCollisionTest private void insert(String key) { - Mutation rm; - rm = new Mutation(KEYSPACE1, ByteBufferUtil.bytes(key)); - rm.add(CF, Util.cellname("column"), ByteBufferUtil.bytes("asdf"), 0); - rm.applyUnsafe(); + RowUpdateBuilder builder = new RowUpdateBuilder(Schema.instance.getCFMetaData(KEYSPACE1, CF), FBUtilities.timestampMicros(), key); + builder.clustering("c").add("val", "asdf").build().applyUnsafe(); } static class BigIntegerToken extends ComparableObjectToken diff --git a/test/unit/org/apache/cassandra/dht/Murmur3PartitionerTest.java b/test/unit/org/apache/cassandra/dht/Murmur3PartitionerTest.java index 9f330d3bf7..19aba40fff 100644 --- a/test/unit/org/apache/cassandra/dht/Murmur3PartitionerTest.java +++ b/test/unit/org/apache/cassandra/dht/Murmur3PartitionerTest.java @@ -1,5 +1,4 @@ /* - * * Licensed to the Apache Software Foundation (ASF) under one * or more contributor license agreements. See the NOTICE file * distributed with this work for additional information @@ -8,15 +7,13 @@ * "License"); you may not use this file except in compliance * with the License. You may obtain a copy of the License at * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. + * http://www.apache.org/licenses/LICENSE-2.0 * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. */ package org.apache.cassandra.dht; diff --git a/test/unit/org/apache/cassandra/dht/OrderPreservingPartitionerTest.java b/test/unit/org/apache/cassandra/dht/OrderPreservingPartitionerTest.java index 0449258427..57f33e79d4 100644 --- a/test/unit/org/apache/cassandra/dht/OrderPreservingPartitionerTest.java +++ b/test/unit/org/apache/cassandra/dht/OrderPreservingPartitionerTest.java @@ -1,21 +1,20 @@ /* -* Licensed to the Apache Software Foundation (ASF) under one -* or more contributor license agreements. See the NOTICE file -* distributed with this work for additional information -* regarding copyright ownership. The ASF licenses this file -* to you under the Apache License, Version 2.0 (the -* "License"); you may not use this file except in compliance -* with the License. You may obtain a copy of the License at -* -* http://www.apache.org/licenses/LICENSE-2.0 -* -* Unless required by applicable law or agreed to in writing, -* software distributed under the License is distributed on an -* "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -* KIND, either express or implied. See the License for the -* specific language governing permissions and limitations -* under the License. -*/ + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ package org.apache.cassandra.dht; import java.io.IOException; diff --git a/test/unit/org/apache/cassandra/dht/PartitionerTestCase.java b/test/unit/org/apache/cassandra/dht/PartitionerTestCase.java index 8080a0c0d3..887c481b3b 100644 --- a/test/unit/org/apache/cassandra/dht/PartitionerTestCase.java +++ b/test/unit/org/apache/cassandra/dht/PartitionerTestCase.java @@ -1,30 +1,34 @@ /* -* Licensed to the Apache Software Foundation (ASF) under one -* or more contributor license agreements. See the NOTICE file -* distributed with this work for additional information -* regarding copyright ownership. The ASF licenses this file -* to you under the Apache License, Version 2.0 (the -* "License"); you may not use this file except in compliance -* with the License. You may obtain a copy of the License at -* -* http://www.apache.org/licenses/LICENSE-2.0 -* -* Unless required by applicable law or agreed to in writing, -* software distributed under the License is distributed on an -* "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -* KIND, either express or implied. See the License for the -* specific language governing permissions and limitations -* under the License. -*/ + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ package org.apache.cassandra.dht; import java.nio.ByteBuffer; -import java.util.*; +import java.util.ArrayList; +import java.util.Collections; +import java.util.List; +import java.util.Map; +import java.util.Random; -import org.apache.cassandra.service.StorageService; import org.junit.Before; import org.junit.Test; +import org.apache.cassandra.service.StorageService; + import static org.junit.Assert.assertEquals; import static org.junit.Assert.fail; diff --git a/test/unit/org/apache/cassandra/dht/RandomPartitionerTest.java b/test/unit/org/apache/cassandra/dht/RandomPartitionerTest.java index 6b22617d69..5e6864438a 100644 --- a/test/unit/org/apache/cassandra/dht/RandomPartitionerTest.java +++ b/test/unit/org/apache/cassandra/dht/RandomPartitionerTest.java @@ -1,6 +1,4 @@ -package org.apache.cassandra.dht; /* - * * Licensed to the Apache Software Foundation (ASF) under one * or more contributor license agreements. See the NOTICE file * distributed with this work for additional information @@ -9,17 +7,17 @@ package org.apache.cassandra.dht; * "License"); you may not use this file except in compliance * with the License. You may obtain a copy of the License at * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. + * http://www.apache.org/licenses/LICENSE-2.0 * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. */ +package org.apache.cassandra.dht; + public class RandomPartitionerTest extends PartitionerTestCase { public void initPartitioner() diff --git a/test/unit/org/apache/cassandra/dht/RangeTest.java b/test/unit/org/apache/cassandra/dht/RangeTest.java index d93356ab94..7888b85660 100644 --- a/test/unit/org/apache/cassandra/dht/RangeTest.java +++ b/test/unit/org/apache/cassandra/dht/RangeTest.java @@ -1,21 +1,20 @@ /* -* Licensed to the Apache Software Foundation (ASF) under one -* or more contributor license agreements. See the NOTICE file -* distributed with this work for additional information -* regarding copyright ownership. The ASF licenses this file -* to you under the Apache License, Version 2.0 (the -* "License"); you may not use this file except in compliance -* with the License. You may obtain a copy of the License at -* -* http://www.apache.org/licenses/LICENSE-2.0 -* -* Unless required by applicable law or agreed to in writing, -* software distributed under the License is distributed on an -* "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -* KIND, either express or implied. See the License for the -* specific language governing permissions and limitations -* under the License. -*/ + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ package org.apache.cassandra.dht; import java.nio.ByteBuffer; @@ -23,14 +22,14 @@ import java.util.HashSet; import java.util.List; import java.util.Set; -import static java.util.Arrays.asList; - import org.apache.commons.lang3.StringUtils; import org.junit.Test; -import org.apache.cassandra.db.RowPosition; -import org.apache.cassandra.dht.RandomPartitioner.BigIntegerToken; -import org.apache.cassandra.dht.ByteOrderedPartitioner.BytesToken; +import org.apache.cassandra.db.PartitionPosition; +import org.apache.cassandra.dht.ByteOrderedPartitioner.BytesToken; +import org.apache.cassandra.dht.RandomPartitioner.BigIntegerToken; + +import static java.util.Arrays.asList; import static org.apache.cassandra.Util.range; @@ -183,7 +182,7 @@ public class RangeTest } @SafeVarargs - static > void assertIntersection(Range one, Range two, Range ... ranges) + static > void assertIntersection(Range one, Range two, Range... ranges) { Set> correct = Range.rangeSet(ranges); Set> result1 = one.intersectionWith(two); @@ -479,7 +478,7 @@ public class RangeTest @Test public void testNormalizeNoop() { - List> l; + List> l; l = asList(range("1", "3"), range("4", "5")); assertNormalize(l, l); @@ -488,7 +487,7 @@ public class RangeTest @Test public void testNormalizeSimpleOverlap() { - List> input, expected; + List> input, expected; input = asList(range("1", "4"), range("3", "5")); expected = asList(range("1", "5")); @@ -502,7 +501,7 @@ public class RangeTest @Test public void testNormalizeSort() { - List> input, expected; + List> input, expected; input = asList(range("4", "5"), range("1", "3")); expected = asList(range("1", "3"), range("4", "5")); @@ -512,7 +511,7 @@ public class RangeTest @Test public void testNormalizeUnwrap() { - List> input, expected; + List> input, expected; input = asList(range("9", "2")); expected = asList(range("", "2"), range("9", "")); @@ -522,7 +521,7 @@ public class RangeTest @Test public void testNormalizeComplex() { - List> input, expected; + List> input, expected; input = asList(range("8", "2"), range("7", "9"), range("4", "5")); expected = asList(range("", "2"), range("4", "5"), range("7", "")); diff --git a/test/unit/org/apache/cassandra/io/compress/CompressedRandomAccessReaderTest.java b/test/unit/org/apache/cassandra/io/compress/CompressedRandomAccessReaderTest.java index 0cf4cfa51c..8aa2fcd056 100644 --- a/test/unit/org/apache/cassandra/io/compress/CompressedRandomAccessReaderTest.java +++ b/test/unit/org/apache/cassandra/io/compress/CompressedRandomAccessReaderTest.java @@ -25,7 +25,8 @@ import java.util.Collections; import java.util.Random; import org.junit.Test; -import org.apache.cassandra.db.composites.SimpleDenseCellNameType; + +import org.apache.cassandra.db.ClusteringComparator; import org.apache.cassandra.db.marshal.BytesType; import org.apache.cassandra.exceptions.ConfigurationException; import org.apache.cassandra.io.sstable.CorruptSSTableException; @@ -66,7 +67,7 @@ public class CompressedRandomAccessReaderTest try { - MetadataCollector sstableMetadataCollector = new MetadataCollector(new SimpleDenseCellNameType(BytesType.instance)); + MetadataCollector sstableMetadataCollector = new MetadataCollector(new ClusteringComparator(BytesType.instance)); CompressedSequentialWriter writer = new CompressedSequentialWriter(f, filename + ".metadata", new CompressionParameters(SnappyCompressor.instance, 32, Collections.emptyMap()), sstableMetadataCollector); for (int i = 0; i < 20; i++) @@ -96,8 +97,8 @@ public class CompressedRandomAccessReaderTest if (f.exists()) f.delete(); File metadata = new File(filename+ ".metadata"); - if (metadata.exists()) - metadata.delete(); + if (metadata.exists()) + metadata.delete(); } } @@ -107,7 +108,7 @@ public class CompressedRandomAccessReaderTest ChannelProxy channel = new ChannelProxy(f); try { - MetadataCollector sstableMetadataCollector = new MetadataCollector(new SimpleDenseCellNameType(BytesType.instance)).replayPosition(null); + MetadataCollector sstableMetadataCollector = new MetadataCollector(new ClusteringComparator(BytesType.instance)).replayPosition(null); SequentialWriter writer = compressed ? new CompressedSequentialWriter(f, filename + ".metadata", new CompressionParameters(SnappyCompressor.instance), sstableMetadataCollector) : SequentialWriter.open(f); @@ -160,7 +161,7 @@ public class CompressedRandomAccessReaderTest File metadata = new File(file.getPath() + ".meta"); metadata.deleteOnExit(); - MetadataCollector sstableMetadataCollector = new MetadataCollector(new SimpleDenseCellNameType(BytesType.instance)).replayPosition(null); + MetadataCollector sstableMetadataCollector = new MetadataCollector(new ClusteringComparator(BytesType.instance)).replayPosition(null); try (SequentialWriter writer = new CompressedSequentialWriter(file, metadata.getPath(), new CompressionParameters(SnappyCompressor.instance), sstableMetadataCollector)) { writer.write(CONTENT.getBytes()); diff --git a/test/unit/org/apache/cassandra/io/compress/CompressedSequentialWriterTest.java b/test/unit/org/apache/cassandra/io/compress/CompressedSequentialWriterTest.java index 27b866db20..aa26dc0ac8 100644 --- a/test/unit/org/apache/cassandra/io/compress/CompressedSequentialWriterTest.java +++ b/test/unit/org/apache/cassandra/io/compress/CompressedSequentialWriterTest.java @@ -31,8 +31,8 @@ import org.junit.After; import org.junit.Test; import junit.framework.Assert; -import org.apache.cassandra.db.composites.CellNames; -import org.apache.cassandra.db.composites.SimpleDenseCellNameType; +import org.apache.cassandra.db.ClusteringComparator; +import org.apache.cassandra.db.marshal.AbstractType; import org.apache.cassandra.db.marshal.BytesType; import org.apache.cassandra.db.marshal.UTF8Type; import org.apache.cassandra.io.sstable.metadata.MetadataCollector; @@ -85,7 +85,8 @@ public class CompressedSequentialWriterTest extends SequentialWriterTest try { - MetadataCollector sstableMetadataCollector = new MetadataCollector(new SimpleDenseCellNameType(BytesType.instance)).replayPosition(null); + MetadataCollector sstableMetadataCollector = new MetadataCollector(new ClusteringComparator(Arrays.>asList(BytesType.instance))).replayPosition(null); + byte[] dataPre = new byte[bytesToTest]; byte[] rawPost = new byte[bytesToTest]; try (CompressedSequentialWriter writer = new CompressedSequentialWriter(f, filename + ".metadata", new CompressionParameters(compressor), sstableMetadataCollector);) @@ -174,7 +175,10 @@ public class CompressedSequentialWriterTest extends SequentialWriterTest private TestableCSW(File file, File offsetsFile) throws IOException { - this(file, offsetsFile, new CompressedSequentialWriter(file, offsetsFile.getPath(), new CompressionParameters(LZ4Compressor.instance, BUFFER_SIZE, new HashMap()), new MetadataCollector(CellNames.fromAbstractType(UTF8Type.instance, false)))); + this(file, offsetsFile, new CompressedSequentialWriter(file, + offsetsFile.getPath(), + new CompressionParameters(LZ4Compressor.instance, BUFFER_SIZE, new HashMap()), + new MetadataCollector(new ClusteringComparator(UTF8Type.instance)))); } private TestableCSW(File file, File offsetsFile, CompressedSequentialWriter sw) throws IOException diff --git a/test/unit/org/apache/cassandra/io/compress/CompressorTest.java b/test/unit/org/apache/cassandra/io/compress/CompressorTest.java index 8ff0074304..ba53718358 100644 --- a/test/unit/org/apache/cassandra/io/compress/CompressorTest.java +++ b/test/unit/org/apache/cassandra/io/compress/CompressorTest.java @@ -87,7 +87,7 @@ public class CompressorTest assertEquals(decompressedLength, len); assertArrayEquals(Arrays.copyOfRange(data, off, off + len), - Arrays.copyOfRange(restored, restoreOffset, restoreOffset + decompressedLength)); + Arrays.copyOfRange(restored, restoreOffset, restoreOffset + decompressedLength)); } public void testArrayUncompress(byte[] data) throws IOException diff --git a/test/unit/org/apache/cassandra/io/sstable/BigTableWriterTest.java b/test/unit/org/apache/cassandra/io/sstable/BigTableWriterTest.java index dfb55a1bd5..56833565ff 100644 --- a/test/unit/org/apache/cassandra/io/sstable/BigTableWriterTest.java +++ b/test/unit/org/apache/cassandra/io/sstable/BigTableWriterTest.java @@ -21,20 +21,19 @@ package org.apache.cassandra.io.sstable; import java.io.File; import java.io.IOException; -import org.junit.After; import org.junit.BeforeClass; import junit.framework.Assert; import org.apache.cassandra.SchemaLoader; -import org.apache.cassandra.Util; +import org.apache.cassandra.UpdateBuilder; import org.apache.cassandra.config.KSMetaData; -import org.apache.cassandra.db.ArrayBackedSortedColumns; import org.apache.cassandra.db.ColumnFamilyStore; import org.apache.cassandra.db.Keyspace; +import org.apache.cassandra.db.SerializationHeader; +import org.apache.cassandra.db.marshal.*; +import org.apache.cassandra.db.rows.RowStats; import org.apache.cassandra.io.sstable.format.SSTableWriter; import org.apache.cassandra.locator.SimpleStrategy; -import org.apache.cassandra.service.StorageService; -import org.apache.cassandra.utils.ByteBufferUtil; import org.apache.cassandra.utils.concurrent.AbstractTransactionalTest; public class BigTableWriterTest extends AbstractTransactionalTest @@ -51,7 +50,7 @@ public class BigTableWriterTest extends AbstractTransactionalTest SchemaLoader.createKeyspace(KEYSPACE1, SimpleStrategy.class, KSMetaData.optsWithRF(1), - SchemaLoader.standardCFMD(KEYSPACE1, CF_STANDARD)); + SchemaLoader.standardCFMD(KEYSPACE1, CF_STANDARD, 0, Int32Type.instance, AsciiType.instance, Int32Type.instance)); cfs = Keyspace.open(KEYSPACE1).getColumnFamilyStore(CF_STANDARD); } @@ -73,7 +72,7 @@ public class BigTableWriterTest extends AbstractTransactionalTest private TestableBTW(String file) throws IOException { - this(file, SSTableWriter.create(file, 0, 0)); + this(file, SSTableWriter.create(file, 0, 0, new SerializationHeader(cfs.metadata, cfs.metadata.partitionColumns(), RowStats.NO_STATS))); } private TestableBTW(String file, SSTableWriter sw) throws IOException @@ -82,11 +81,14 @@ public class BigTableWriterTest extends AbstractTransactionalTest this.file = new File(file); this.descriptor = Descriptor.fromFilename(file); this.writer = sw; - ArrayBackedSortedColumns cf = ArrayBackedSortedColumns.factory.create(cfs.metadata); - for (int i = 0; i < 10; i++) - cf.addColumn(Util.cellname(i), SSTableRewriterTest.random(0, 1000), 1); + for (int i = 0; i < 100; i++) - writer.append(StorageService.getPartitioner().decorateKey(ByteBufferUtil.bytes(i)), cf); + { + UpdateBuilder update = UpdateBuilder.create(cfs.metadata, i); + for (int j = 0; j < 10; j++) + update.newRow(j).add("val", SSTableRewriterTest.random(0, 1000)); + writer.append(update.build().unfilteredIterator()); + } } protected void assertInProgress() throws Exception diff --git a/test/unit/org/apache/cassandra/io/sstable/CQLSSTableWriterClientTest.java b/test/unit/org/apache/cassandra/io/sstable/CQLSSTableWriterClientTest.java index ad2d876e3d..dab88c8cb0 100644 --- a/test/unit/org/apache/cassandra/io/sstable/CQLSSTableWriterClientTest.java +++ b/test/unit/org/apache/cassandra/io/sstable/CQLSSTableWriterClientTest.java @@ -20,11 +20,12 @@ package org.apache.cassandra.io.sstable; import java.io.File; import java.io.FilenameFilter; import java.io.IOException; -import java.util.Arrays; import com.google.common.io.Files; - -import org.junit.*; +import org.junit.After; +import org.junit.AfterClass; +import org.junit.Before; +import org.junit.Test; import org.apache.cassandra.config.Config; import org.apache.cassandra.exceptions.InvalidRequestException; @@ -32,8 +33,6 @@ import org.apache.cassandra.io.util.FileUtils; import static org.junit.Assert.assertEquals; -import static org.junit.Assert.assertTrue; - public class CQLSSTableWriterClientTest { private File testDirectory; diff --git a/test/unit/org/apache/cassandra/io/sstable/CQLSSTableWriterTest.java b/test/unit/org/apache/cassandra/io/sstable/CQLSSTableWriterTest.java index 70d70ceb8b..7bc21ee516 100644 --- a/test/unit/org/apache/cassandra/io/sstable/CQLSSTableWriterTest.java +++ b/test/unit/org/apache/cassandra/io/sstable/CQLSSTableWriterTest.java @@ -26,7 +26,6 @@ import java.util.UUID; import com.google.common.collect.ImmutableMap; import com.google.common.io.Files; - import org.junit.AfterClass; import org.junit.BeforeClass; import org.junit.Test; @@ -86,9 +85,10 @@ public class CQLSSTableWriterTest .using(insert).build(); writer.addRow(0, "test1", 24); - writer.addRow(1, "test2", null); + writer.addRow(1, "test2", 44); writer.addRow(2, "test3", 42); writer.addRow(ImmutableMap.of("k", 3, "v2", 12)); + writer.close(); SSTableLoader loader = new SSTableLoader(dataDir, new SSTableLoader.Client() @@ -103,9 +103,9 @@ public class CQLSSTableWriterTest setPartitioner(StorageService.getPartitioner()); } - public CFMetaData getTableMetadata(String tableName) + public CFMetaData getTableMetadata(String cfName) { - return Schema.instance.getCFMetaData(keyspace, tableName); + return Schema.instance.getCFMetaData(keyspace, cfName); } }, new OutputHandler.SystemOutput(false, false)); @@ -125,7 +125,8 @@ public class CQLSSTableWriterTest row = iter.next(); assertEquals(1, row.getInt("k")); assertEquals("test2", row.getString("v1")); - assertFalse(row.has("v2")); + //assertFalse(row.has("v2")); + assertEquals(44, row.getInt("v2")); row = iter.next(); assertEquals(2, row.getInt("k")); @@ -151,12 +152,10 @@ public class CQLSSTableWriterTest File dataDir = new File(tempdir.getAbsolutePath() + File.separator + KS + File.separator + TABLE); assert dataDir.mkdirs(); String schema = "CREATE TABLE ks.test (" - + " k int," - + " c int," - + " v blob," - + " PRIMARY KEY (k,c)" + + " k int PRIMARY KEY," + + " v blob" + ")"; - String insert = "INSERT INTO ks.test (k, c, v) VALUES (?, ?, ?)"; + String insert = "INSERT INTO ks.test (k, v) VALUES (?, ?)"; CQLSSTableWriter writer = CQLSSTableWriter.builder() .inDirectory(dataDir) .forTable(schema) @@ -167,8 +166,8 @@ public class CQLSSTableWriterTest ByteBuffer val = ByteBuffer.allocate(1024 * 1050); - writer.addRow(0, 0, val); - writer.addRow(0, 1, val); + writer.addRow(0, val); + writer.addRow(1, val); writer.close(); FilenameFilter filterDataFiles = new FilenameFilter() @@ -293,9 +292,9 @@ public class CQLSSTableWriterTest setPartitioner(StorageService.getPartitioner()); } - public CFMetaData getTableMetadata(String tableName) + public CFMetaData getTableMetadata(String cfName) { - return Schema.instance.getCFMetaData(keyspace, tableName); + return Schema.instance.getCFMetaData(keyspace, cfName); } }, new OutputHandler.SystemOutput(false, false)); diff --git a/test/unit/org/apache/cassandra/io/sstable/DescriptorTest.java b/test/unit/org/apache/cassandra/io/sstable/DescriptorTest.java index 70ab8ba3fd..ceb0f9c908 100644 --- a/test/unit/org/apache/cassandra/io/sstable/DescriptorTest.java +++ b/test/unit/org/apache/cassandra/io/sstable/DescriptorTest.java @@ -21,16 +21,17 @@ import java.io.File; import java.io.IOException; import java.util.UUID; -import org.apache.cassandra.io.sstable.format.SSTableFormat; import org.apache.commons.lang3.StringUtils; import org.junit.Assert; import org.junit.Test; import org.apache.cassandra.db.Directories; +import org.apache.cassandra.io.sstable.format.SSTableFormat; import org.apache.cassandra.utils.ByteBufferUtil; import org.apache.cassandra.utils.Pair; -import static org.junit.Assert.*; +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertNull; public class DescriptorTest { diff --git a/test/unit/org/apache/cassandra/io/sstable/IndexHelperTest.java b/test/unit/org/apache/cassandra/io/sstable/IndexHelperTest.java index 59ef4c439e..f4f7de3cfb 100644 --- a/test/unit/org/apache/cassandra/io/sstable/IndexHelperTest.java +++ b/test/unit/org/apache/cassandra/io/sstable/IndexHelperTest.java @@ -19,33 +19,42 @@ package org.apache.cassandra.io.sstable; import java.util.ArrayList; +import java.util.Collections; import java.util.List; -import static org.junit.Assert.*; - import org.junit.Test; import org.apache.cassandra.Util; -import org.apache.cassandra.db.composites.*; +import org.apache.cassandra.db.ClusteringComparator; +import org.apache.cassandra.db.ClusteringPrefix; +import org.apache.cassandra.db.SimpleDeletionTime; +import org.apache.cassandra.db.marshal.AbstractType; import org.apache.cassandra.db.marshal.IntegerType; +import org.apache.cassandra.db.marshal.LongType; +import org.apache.cassandra.utils.FBUtilities; + import static org.apache.cassandra.io.sstable.IndexHelper.IndexInfo; +import static org.junit.Assert.assertEquals; public class IndexHelperTest { - private static CellName cn(long l) + + private static ClusteringComparator comp = new ClusteringComparator(Collections.>singletonList(LongType.instance)); + private static ClusteringPrefix cn(long l) { - return Util.cellname(l); + return Util.clustering(comp, l); } @Test public void testIndexHelper() { - List indexes = new ArrayList(); - indexes.add(new IndexInfo(cn(0L), cn(5L), 0, 0)); - indexes.add(new IndexInfo(cn(10L), cn(15L), 0, 0)); - indexes.add(new IndexInfo(cn(20L), cn(25L), 0, 0)); + SimpleDeletionTime deletionInfo = new SimpleDeletionTime(FBUtilities.timestampMicros(), FBUtilities.nowInSeconds()); + + List indexes = new ArrayList<>(); + indexes.add(new IndexInfo(cn(0L), cn(5L), 0, 0, deletionInfo)); + indexes.add(new IndexInfo(cn(10L), cn(15L), 0, 0, deletionInfo)); + indexes.add(new IndexInfo(cn(20L), cn(25L), 0, 0, deletionInfo)); - CellNameType comp = new SimpleDenseCellNameType(IntegerType.instance); assertEquals(0, IndexHelper.indexFor(cn(-1L), indexes, comp, false, -1)); assertEquals(0, IndexHelper.indexFor(cn(5L), indexes, comp, false, -1)); @@ -55,16 +64,17 @@ public class IndexHelperTest assertEquals(3, IndexHelper.indexFor(cn(100L), indexes, comp, false, 0)); assertEquals(3, IndexHelper.indexFor(cn(100L), indexes, comp, false, 1)); assertEquals(3, IndexHelper.indexFor(cn(100L), indexes, comp, false, 2)); - assertEquals(-1, IndexHelper.indexFor(cn(100L), indexes, comp, false, 3)); + assertEquals(3, IndexHelper.indexFor(cn(100L), indexes, comp, false, 3)); assertEquals(-1, IndexHelper.indexFor(cn(-1L), indexes, comp, true, -1)); - assertEquals(0, IndexHelper.indexFor(cn(5L), indexes, comp, true, -1)); - assertEquals(1, IndexHelper.indexFor(cn(17L), indexes, comp, true, -1)); - assertEquals(2, IndexHelper.indexFor(cn(100L), indexes, comp, true, -1)); - assertEquals(0, IndexHelper.indexFor(cn(100L), indexes, comp, true, 0)); - assertEquals(1, IndexHelper.indexFor(cn(12L), indexes, comp, true, -1)); + assertEquals(0, IndexHelper.indexFor(cn(5L), indexes, comp, true, 3)); + assertEquals(0, IndexHelper.indexFor(cn(5L), indexes, comp, true, 2)); + assertEquals(1, IndexHelper.indexFor(cn(17L), indexes, comp, true, 3)); + assertEquals(2, IndexHelper.indexFor(cn(100L), indexes, comp, true, 3)); + assertEquals(2, IndexHelper.indexFor(cn(100L), indexes, comp, true, 4)); + assertEquals(1, IndexHelper.indexFor(cn(12L), indexes, comp, true, 3)); + assertEquals(1, IndexHelper.indexFor(cn(12L), indexes, comp, true, 2)); assertEquals(1, IndexHelper.indexFor(cn(100L), indexes, comp, true, 1)); assertEquals(2, IndexHelper.indexFor(cn(100L), indexes, comp, true, 2)); - assertEquals(-1, IndexHelper.indexFor(cn(100L), indexes, comp, true, 4)); } } diff --git a/test/unit/org/apache/cassandra/io/sstable/IndexSummaryManagerTest.java b/test/unit/org/apache/cassandra/io/sstable/IndexSummaryManagerTest.java index 5e46b8ed1f..4037cb7e70 100644 --- a/test/unit/org/apache/cassandra/io/sstable/IndexSummaryManagerTest.java +++ b/test/unit/org/apache/cassandra/io/sstable/IndexSummaryManagerTest.java @@ -15,21 +15,29 @@ * See the License for the specific language governing permissions and * limitations under the License. */ + package org.apache.cassandra.io.sstable; import java.io.IOException; import java.nio.ByteBuffer; -import java.util.*; -import java.util.concurrent.*; +import java.util.ArrayList; +import java.util.Arrays; +import java.util.Collections; +import java.util.Comparator; +import java.util.List; +import java.util.Map; +import java.util.concurrent.ExecutionException; +import java.util.concurrent.ExecutorService; +import java.util.concurrent.Executors; +import java.util.concurrent.Future; +import java.util.concurrent.TimeUnit; import java.util.concurrent.atomic.AtomicBoolean; -import org.apache.cassandra.io.sstable.format.SSTableReader; import org.junit.After; import org.junit.Before; import org.junit.BeforeClass; import org.junit.Test; import org.junit.runner.RunWith; - import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -39,14 +47,20 @@ import org.apache.cassandra.SchemaLoader; import org.apache.cassandra.Util; import org.apache.cassandra.cache.CachingOptions; import org.apache.cassandra.config.KSMetaData; -import org.apache.cassandra.db.*; +import org.apache.cassandra.db.ColumnFamilyStore; +import org.apache.cassandra.db.DecoratedKey; +import org.apache.cassandra.db.Keyspace; +import org.apache.cassandra.db.RowUpdateBuilder; +import org.apache.cassandra.db.rows.*; import org.apache.cassandra.db.compaction.CompactionManager; import org.apache.cassandra.db.compaction.OperationType; -import org.apache.cassandra.db.filter.QueryFilter; +import org.apache.cassandra.db.partitions.ArrayBackedPartition; import org.apache.cassandra.exceptions.ConfigurationException; +import org.apache.cassandra.io.sstable.format.SSTableReader; import org.apache.cassandra.locator.SimpleStrategy; import org.apache.cassandra.db.lifecycle.LifecycleTransaction; import org.apache.cassandra.metrics.RestorableMeter; +import org.apache.cassandra.utils.ByteBufferUtil; import static com.google.common.collect.ImmutableMap.of; import static java.util.Arrays.asList; @@ -58,6 +72,8 @@ import static org.junit.Assert.assertEquals; import static org.junit.Assert.assertNotNull; import static org.junit.Assert.assertNull; import static org.junit.Assert.assertTrue; +import static org.junit.Assert.assertFalse; + @RunWith(OrderedJUnit4ClassRunner.class) public class IndexSummaryManagerTest @@ -88,7 +104,7 @@ public class IndexSummaryManagerTest .minIndexInterval(8) .maxIndexInterval(256) .caching(CachingOptions.NONE) - ); + ); } @Before @@ -138,17 +154,15 @@ public class IndexSummaryManagerTest return sstables; } - private void validateData(ColumnFamilyStore cfs, int numRows) + private void validateData(ColumnFamilyStore cfs, int numPartition) { - for (int i = 0; i < numRows; i++) + for (int i = 0; i < numPartition; i++) { - DecoratedKey key = Util.dk(String.format("%3d", i)); - QueryFilter filter = QueryFilter.getIdentityFilter(key, cfs.getColumnFamilyName(), System.currentTimeMillis()); - ColumnFamily row = cfs.getColumnFamily(filter); - assertNotNull(row); - Cell cell = row.getColumn(Util.cellname("column")); + Row row = Util.getOnlyRowUnfiltered(Util.cmd(cfs, String.format("%3d", i)).build()); + Cell cell = row.getCell(cfs.metadata.getColumnDefinition(ByteBufferUtil.bytes("val"))); assertNotNull(cell); assertEquals(100, cell.value().array().length); + } } @@ -160,7 +174,7 @@ public class IndexSummaryManagerTest } }; - private void createSSTables(String ksname, String cfname, int numSSTables, int numRows) + private void createSSTables(String ksname, String cfname, int numSSTables, int numPartition) { Keyspace keyspace = Keyspace.open(ksname); ColumnFamilyStore cfs = keyspace.getColumnFamilyStore(cfname); @@ -171,12 +185,15 @@ public class IndexSummaryManagerTest ByteBuffer value = ByteBuffer.wrap(new byte[100]); for (int sstable = 0; sstable < numSSTables; sstable++) { - for (int row = 0; row < numRows; row++) + for (int p = 0; p < numPartition; p++) { - DecoratedKey key = Util.dk(String.format("%3d", row)); - Mutation rm = new Mutation(ksname, key.getKey()); - rm.add(cfname, Util.cellname("column"), value, 0); - rm.applyUnsafe(); + + String key = String.format("%3d", p); + new RowUpdateBuilder(cfs.metadata, 0, key) + .clustering("column") + .add("val", value) + .build() + .applyUnsafe(); } futures.add(cfs.forceFlush()); } @@ -191,7 +208,7 @@ public class IndexSummaryManagerTest } } assertEquals(numSSTables, cfs.getSSTables().size()); - validateData(cfs, numRows); + validateData(cfs, numPartition); } @Test @@ -494,11 +511,14 @@ public class IndexSummaryManagerTest int numRows = 256; for (int row = 0; row < numRows; row++) { - DecoratedKey key = Util.dk(String.valueOf(row)); - Mutation rm = new Mutation(ksname, key.getKey()); - rm.add(cfname, Util.cellname("column"), value, 0); - rm.applyUnsafe(); + String key = String.format("%3d", row); + new RowUpdateBuilder(cfs.metadata, 0, key) + .clustering("column") + .add("val", value) + .build() + .applyUnsafe(); } + cfs.forceBlockingFlush(); List sstables = new ArrayList<>(cfs.getSSTables()); @@ -557,10 +577,12 @@ public class IndexSummaryManagerTest { for (int row = 0; row < numRows; row++) { - DecoratedKey key = Util.dk(String.valueOf(row)); - Mutation rm = new Mutation(ksname, key.getKey()); - rm.add(cfname, Util.cellname("column"), value, 0); - rm.applyUnsafe(); + String key = String.format("%3d", row); + new RowUpdateBuilder(cfs.metadata, 0, key) + .clustering("column") + .add("val", value) + .build() + .applyUnsafe(); } cfs.forceBlockingFlush(); } diff --git a/test/unit/org/apache/cassandra/io/sstable/LegacySSTableTest.java b/test/unit/org/apache/cassandra/io/sstable/LegacySSTableTest.java index f4b9617f9a..67c5b51726 100644 --- a/test/unit/org/apache/cassandra/io/sstable/LegacySSTableTest.java +++ b/test/unit/org/apache/cassandra/io/sstable/LegacySSTableTest.java @@ -1,21 +1,20 @@ /* -* Licensed to the Apache Software Foundation (ASF) under one -* or more contributor license agreements. See the NOTICE file -* distributed with this work for additional information -* regarding copyright ownership. The ASF licenses this file -* to you under the Apache License, Version 2.0 (the -* "License"); you may not use this file except in compliance -* with the License. You may obtain a copy of the License at -* -* http://www.apache.org/licenses/LICENSE-2.0 -* -* Unless required by applicable law or agreed to in writing, -* software distributed under the License is distributed on an -* "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -* KIND, either express or implied. See the License for the -* specific language governing permissions and limitations -* under the License. -*/ + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ package org.apache.cassandra.io.sstable; @@ -26,26 +25,26 @@ import java.util.HashSet; import java.util.List; import java.util.Set; -import org.apache.cassandra.db.columniterator.OnDiskAtomIterator; -import org.apache.cassandra.io.sstable.format.SSTableFormat; -import org.apache.cassandra.io.sstable.format.SSTableReader; -import org.apache.cassandra.io.sstable.format.Version; import org.junit.BeforeClass; import org.junit.Test; import org.apache.cassandra.SchemaLoader; import org.apache.cassandra.Util; import org.apache.cassandra.config.KSMetaData; -import org.apache.cassandra.db.ColumnFamily; +import org.apache.cassandra.config.CFMetaData; import org.apache.cassandra.db.ColumnFamilyStore; -import org.apache.cassandra.db.DecoratedKey; -import org.apache.cassandra.db.DeletionInfo; +import org.apache.cassandra.db.DeletionTime; import org.apache.cassandra.db.Keyspace; -import org.apache.cassandra.db.composites.CellNameType; +import org.apache.cassandra.db.rows.SliceableUnfilteredRowIterator; +import org.apache.cassandra.db.filter.ColumnFilter; +import org.apache.cassandra.db.marshal.BytesType; import org.apache.cassandra.dht.IPartitioner; import org.apache.cassandra.dht.Range; import org.apache.cassandra.dht.Token; import org.apache.cassandra.exceptions.ConfigurationException; +import org.apache.cassandra.io.sstable.format.SSTableFormat; +import org.apache.cassandra.io.sstable.format.SSTableReader; +import org.apache.cassandra.io.sstable.format.Version; import org.apache.cassandra.locator.SimpleStrategy; import org.apache.cassandra.service.StorageService; import org.apache.cassandra.streaming.StreamPlan; @@ -53,6 +52,8 @@ import org.apache.cassandra.streaming.StreamSession; import org.apache.cassandra.utils.ByteBufferUtil; import org.apache.cassandra.utils.FBUtilities; +import static org.apache.cassandra.utils.ByteBufferUtil.bytes; + /** * Tests backwards compatibility for SSTables */ @@ -69,10 +70,17 @@ public class LegacySSTableTest public static void defineSchema() throws ConfigurationException { SchemaLoader.prepareServer(); + + CFMetaData metadata = CFMetaData.Builder.createDense(KSNAME, CFNAME, false, false) + .addPartitionKey("key", BytesType.instance) + .addClusteringColumn("column", BytesType.instance) + .addRegularColumn("value", BytesType.instance) + .build(); + SchemaLoader.createKeyspace(KSNAME, SimpleStrategy.class, KSMetaData.optsWithRF(1), - SchemaLoader.standardCFMD(KSNAME, CFNAME)); + metadata); beforeClass(); } @@ -122,8 +130,10 @@ public class LegacySSTableTest StorageService.instance.initServer(); for (File version : LEGACY_SSTABLE_ROOT.listFiles()) - if (Version.validate(version.getName()) && SSTableFormat.Type.LEGACY.info.getVersion(version.getName()).isCompatible()) + { + if (Version.validate(version.getName()) && SSTableFormat.Type.LEGACY.info.getVersion(version.getName()).isCompatibleForStreaming()) testStreaming(version.getName()); + } } private void testStreaming(String version) throws Exception @@ -143,16 +153,15 @@ public class LegacySSTableTest ColumnFamilyStore cfs = Keyspace.open(KSNAME).getColumnFamilyStore(CFNAME); assert cfs.getSSTables().size() == 1; sstable = cfs.getSSTables().iterator().next(); - CellNameType type = sstable.metadata.comparator; for (String keystring : TEST_DATA) { - ByteBuffer key = ByteBufferUtil.bytes(keystring); - OnDiskAtomIterator iter = sstable.iterator(Util.dk(key), FBUtilities.singleton(Util.cellname(key), type)); - ColumnFamily cf = iter.getColumnFamily(); + ByteBuffer key = bytes(keystring); + + SliceableUnfilteredRowIterator iter = sstable.iterator(Util.dk(key), ColumnFilter.selectionBuilder().add(cfs.metadata.getColumnDefinition(bytes("name"))).build(), false, false); // check not deleted (CASSANDRA-6527) - assert cf.deletionInfo().equals(DeletionInfo.live()); - assert iter.next().name().toByteBuffer().equals(key); + assert iter.partitionLevelDeletion().equals(DeletionTime.LIVE); + assert iter.next().clustering().get(0).equals(key); } sstable.selfRef().release(); } @@ -178,15 +187,20 @@ public class LegacySSTableTest { try { + ColumnFamilyStore cfs = Keyspace.open(KSNAME).getColumnFamilyStore(CFNAME); + + SSTableReader reader = SSTableReader.open(getDescriptor(version)); - CellNameType type = reader.metadata.comparator; for (String keystring : TEST_DATA) { - ByteBuffer key = ByteBufferUtil.bytes(keystring); - // confirm that the bloom filter does not reject any keys/names - DecoratedKey dk = reader.partitioner.decorateKey(key); - OnDiskAtomIterator iter = reader.iterator(dk, FBUtilities.singleton(Util.cellname(key), type)); - assert iter.next().name().toByteBuffer().equals(key); + + ByteBuffer key = bytes(keystring); + + SliceableUnfilteredRowIterator iter = reader.iterator(Util.dk(key), ColumnFilter.selection(cfs.metadata.partitionColumns()), false, false); + + // check not deleted (CASSANDRA-6527) + assert iter.partitionLevelDeletion().equals(DeletionTime.LIVE); + assert iter.next().clustering().get(0).equals(key); } // TODO actually test some reads diff --git a/test/unit/org/apache/cassandra/io/sstable/SSTableLoaderTest.java b/test/unit/org/apache/cassandra/io/sstable/SSTableLoaderTest.java index 4a51fbd9f4..4e6c75e00c 100644 --- a/test/unit/org/apache/cassandra/io/sstable/SSTableLoaderTest.java +++ b/test/unit/org/apache/cassandra/io/sstable/SSTableLoaderTest.java @@ -31,7 +31,9 @@ import org.apache.cassandra.config.KSMetaData; import org.apache.cassandra.config.Schema; import org.apache.cassandra.db.DecoratedKey; import org.apache.cassandra.db.Keyspace; -import org.apache.cassandra.db.Row; +import org.apache.cassandra.db.SimpleClustering; +import org.apache.cassandra.db.partitions.*; +import org.apache.cassandra.db.marshal.AsciiType; import org.apache.cassandra.dht.Range; import org.apache.cassandra.dht.Token; import org.apache.cassandra.locator.SimpleStrategy; @@ -70,15 +72,18 @@ public class SSTableLoaderTest File dataDir = new File(tempdir.getAbsolutePath() + File.separator + KEYSPACE1 + File.separator + CF_STANDARD); assert dataDir.mkdirs(); CFMetaData cfmeta = Schema.instance.getCFMetaData(KEYSPACE1, CF_STANDARD); - DecoratedKey key = Util.dk("key1"); - - try (SSTableSimpleUnsortedWriter writer = new SSTableSimpleUnsortedWriter(dataDir, - cfmeta, - StorageService.getPartitioner(), - 1)) + + String schema = "CREATE TABLE %s.%s (key ascii, name ascii, val ascii, val1 ascii, PRIMARY KEY (key, name))"; + String query = "INSERT INTO %s.%s (key, name, val) VALUES (?, ?, ?)"; + ; + try (CQLSSTableWriter writer = CQLSSTableWriter.builder() + .inDirectory(dataDir) + .withPartitioner(StorageService.getPartitioner()) + .forTable(String.format(schema, KEYSPACE1, CF_STANDARD)) + .using(String.format(query, KEYSPACE1, CF_STANDARD)) + .build()) { - writer.newRow(key.getKey()); - writer.addColumn(ByteBufferUtil.bytes("col1"), ByteBufferUtil.bytes(100), 1); + writer.addRow("key1", "col1", "100"); } SSTableLoader loader = new SSTableLoader(dataDir, new SSTableLoader.Client() @@ -101,9 +106,12 @@ public class SSTableLoaderTest loader.stream().get(); - List rows = Util.getRangeSlice(Keyspace.open(KEYSPACE1).getColumnFamilyStore(CF_STANDARD)); - assertEquals(1, rows.size()); - assertEquals(key, rows.get(0).key); - assertEquals(ByteBufferUtil.bytes(100), rows.get(0).cf.getColumn(Util.cellname("col1")).value()); + List partitions = Util.getAll(Util.cmd(Keyspace.open(KEYSPACE1).getColumnFamilyStore(CF_STANDARD)).build()); + + assertEquals(1, partitions.size()); + assertEquals("key1", AsciiType.instance.getString(partitions.get(0).partitionKey().getKey())); + assertEquals(ByteBufferUtil.bytes("100"), partitions.get(0).getRow(new SimpleClustering(ByteBufferUtil.bytes("col1"))) + .getCell(cfmeta.getColumnDefinition(ByteBufferUtil.bytes("val"))) + .value()); } } diff --git a/test/unit/org/apache/cassandra/io/sstable/SSTableMetadataTest.java b/test/unit/org/apache/cassandra/io/sstable/SSTableMetadataTest.java index f952b91fce..31b03dbc0b 100644 --- a/test/unit/org/apache/cassandra/io/sstable/SSTableMetadataTest.java +++ b/test/unit/org/apache/cassandra/io/sstable/SSTableMetadataTest.java @@ -1,5 +1,4 @@ /* - * * Licensed to the Apache Software Foundation (ASF) under one * or more contributor license agreements. See the NOTICE file * distributed with this work for additional information @@ -7,16 +6,14 @@ * to you under the Apache License, Version 2.0 (the * "License"); you may not use this file except in compliance * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - * + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. */ package org.apache.cassandra.io.sstable; @@ -27,7 +24,6 @@ import java.util.Arrays; import java.util.List; import java.util.concurrent.ExecutionException; -import org.apache.cassandra.io.sstable.format.SSTableReader; import org.junit.BeforeClass; import org.junit.Test; @@ -35,20 +31,29 @@ import org.apache.cassandra.SchemaLoader; import org.apache.cassandra.Util; import org.apache.cassandra.config.CFMetaData; import org.apache.cassandra.config.KSMetaData; -import org.apache.cassandra.db.*; -import org.apache.cassandra.db.composites.*; +import org.apache.cassandra.db.ClusteringComparator; +import org.apache.cassandra.db.ColumnFamilyStore; +import org.apache.cassandra.db.DecoratedKey; +import org.apache.cassandra.db.Keyspace; +import org.apache.cassandra.db.Mutation; +import org.apache.cassandra.db.RowUpdateBuilder; import org.apache.cassandra.db.context.CounterContext; -import org.apache.cassandra.db.marshal.*; +import org.apache.cassandra.db.marshal.AbstractType; +import org.apache.cassandra.db.marshal.AsciiType; +import org.apache.cassandra.db.marshal.BytesType; +import org.apache.cassandra.db.marshal.CompositeType; +import org.apache.cassandra.db.marshal.CounterColumnType; +import org.apache.cassandra.db.marshal.IntegerType; +import org.apache.cassandra.io.sstable.format.SSTableReader; import org.apache.cassandra.locator.SimpleStrategy; import org.apache.cassandra.utils.ByteBufferUtil; import org.apache.cassandra.utils.CounterId; +import static org.apache.cassandra.Util.getBytes; import static org.junit.Assert.assertEquals; import static org.junit.Assert.assertFalse; import static org.junit.Assert.assertTrue; -import static org.apache.cassandra.Util.cellname; - public class SSTableMetadataTest { public static final String KEYSPACE1 = "SSTableMetadataTest"; @@ -61,7 +66,6 @@ public class SSTableMetadataTest @BeforeClass public static void defineSchema() throws Exception { - AbstractType compositeMaxMin = CompositeType.getInstance(Arrays.asList(new AbstractType[]{BytesType.instance, IntegerType.instance})); SchemaLoader.prepareServer(); SchemaLoader.createKeyspace(KEYSPACE1, SimpleStrategy.class, @@ -69,8 +73,12 @@ public class SSTableMetadataTest SchemaLoader.standardCFMD(KEYSPACE1, CF_STANDARD), SchemaLoader.standardCFMD(KEYSPACE1, CF_STANDARD2), SchemaLoader.standardCFMD(KEYSPACE1, CF_STANDARD3), - SchemaLoader.standardCFMD(KEYSPACE1, CF_STANDARDCOMPOSITE2, compositeMaxMin), - SchemaLoader.standardCFMD(KEYSPACE1, CF_COUNTER1).defaultValidator(CounterColumnType.instance)); + CFMetaData.Builder.create(KEYSPACE1, CF_STANDARDCOMPOSITE2) + .addPartitionKey("key", AsciiType.instance) + .addClusteringColumn("name", AsciiType.instance) + .addClusteringColumn("int", IntegerType.instance) + .addRegularColumn("val", AsciiType.instance).build(), + SchemaLoader.counterCFMD(KEYSPACE1, CF_COUNTER1)); } @Test @@ -82,20 +90,22 @@ public class SSTableMetadataTest for(int i = 0; i < 10; i++) { DecoratedKey key = Util.dk(Integer.toString(i)); - Mutation rm = new Mutation(KEYSPACE1, key.getKey()); for (int j = 0; j < 10; j++) - rm.add("Standard1", cellname(Integer.toString(j)), - ByteBufferUtil.EMPTY_BYTE_BUFFER, - timestamp, - 10 + j); - rm.applyUnsafe(); + new RowUpdateBuilder(store.metadata, timestamp, 10 + j, Integer.toString(i)) + .clustering(Integer.toString(j)) + .add("val", ByteBufferUtil.EMPTY_BYTE_BUFFER) + .build() + .applyUnsafe(); + } - Mutation rm = new Mutation(KEYSPACE1, Util.dk("longttl").getKey()); - rm.add("Standard1", cellname("col"), - ByteBufferUtil.EMPTY_BYTE_BUFFER, - timestamp, - 10000); - rm.applyUnsafe(); + + new RowUpdateBuilder(store.metadata, timestamp, 10000, "longttl") + .clustering("col") + .add("val", ByteBufferUtil.EMPTY_BYTE_BUFFER) + .build() + .applyUnsafe(); + + store.forceBlockingFlush(); assertEquals(1, store.getSSTables().size()); int ttltimestamp = (int)(System.currentTimeMillis()/1000); @@ -106,12 +116,14 @@ public class SSTableMetadataTest assertEquals(ttltimestamp + 10000, firstDelTime, 10); } - rm = new Mutation(KEYSPACE1, Util.dk("longttl2").getKey()); - rm.add("Standard1", cellname("col"), - ByteBufferUtil.EMPTY_BYTE_BUFFER, - timestamp, - 20000); - rm.applyUnsafe(); + + new RowUpdateBuilder(store.metadata, timestamp, 20000, "longttl2") + .clustering("col") + .add("val", ByteBufferUtil.EMPTY_BYTE_BUFFER) + .build() + .applyUnsafe(); + + ttltimestamp = (int) (System.currentTimeMillis()/1000); store.forceBlockingFlush(); assertEquals(2, store.getSSTables().size()); @@ -153,17 +165,20 @@ public class SSTableMetadataTest ColumnFamilyStore store = keyspace.getColumnFamilyStore("Standard2"); long timestamp = System.currentTimeMillis(); DecoratedKey key = Util.dk("deletetest"); - Mutation rm = new Mutation(KEYSPACE1, key.getKey()); for (int i = 0; i<5; i++) - rm.add("Standard2", cellname("deletecolumn" + i), - ByteBufferUtil.EMPTY_BYTE_BUFFER, - timestamp, - 100); - rm.add("Standard2", cellname("todelete"), - ByteBufferUtil.EMPTY_BYTE_BUFFER, - timestamp, - 1000); - rm.applyUnsafe(); + new RowUpdateBuilder(store.metadata, timestamp, 100, "deletetest") + .clustering("deletecolumn" + i) + .add("val", ByteBufferUtil.EMPTY_BYTE_BUFFER) + .build() + .applyUnsafe(); + + + new RowUpdateBuilder(store.metadata, timestamp, 1000, "deletetest") + .clustering("todelete") + .add("val", ByteBufferUtil.EMPTY_BYTE_BUFFER) + .build() + .applyUnsafe(); + store.forceBlockingFlush(); assertEquals(1,store.getSSTables().size()); int ttltimestamp = (int) (System.currentTimeMillis()/1000); @@ -173,9 +188,9 @@ public class SSTableMetadataTest firstMaxDelTime = sstable.getSSTableMetadata().maxLocalDeletionTime; assertEquals(ttltimestamp + 1000, firstMaxDelTime, 10); } - rm = new Mutation(KEYSPACE1, key.getKey()); - rm.delete("Standard2", cellname("todelete"), timestamp + 1); - rm.applyUnsafe(); + + RowUpdateBuilder.deleteRow(store.metadata, timestamp + 1, "deletetest", "todelete").applyUnsafe(); + store.forceBlockingFlush(); assertEquals(2,store.getSSTables().size()); boolean foundDelete = false; @@ -203,36 +218,41 @@ public class SSTableMetadataTest ColumnFamilyStore store = keyspace.getColumnFamilyStore("Standard3"); for (int j = 0; j < 8; j++) { - DecoratedKey key = Util.dk("row"+j); - Mutation rm = new Mutation(KEYSPACE1, key.getKey()); + String key = "row" + j; for (int i = 100; i<150; i++) { - rm.add("Standard3", cellname(j + "col" + i), ByteBufferUtil.EMPTY_BYTE_BUFFER, System.currentTimeMillis()); + new RowUpdateBuilder(store.metadata, System.currentTimeMillis(), key) + .clustering(j + "col" + i) + .add("val", ByteBufferUtil.EMPTY_BYTE_BUFFER) + .build() + .applyUnsafe(); } - rm.applyUnsafe(); } store.forceBlockingFlush(); assertEquals(1, store.getSSTables().size()); for (SSTableReader sstable : store.getSSTables()) { - assertEquals(ByteBufferUtil.string(sstable.getSSTableMetadata().minColumnNames.get(0)), "0col100"); - assertEquals(ByteBufferUtil.string(sstable.getSSTableMetadata().maxColumnNames.get(0)), "7col149"); + assertEquals(ByteBufferUtil.string(sstable.getSSTableMetadata().minClusteringValues.get(0)), "0col100"); + assertEquals(ByteBufferUtil.string(sstable.getSSTableMetadata().maxClusteringValues.get(0)), "7col149"); } - DecoratedKey key = Util.dk("row2"); - Mutation rm = new Mutation(KEYSPACE1, key.getKey()); + String key = "row2"; + for (int i = 101; i<299; i++) { - rm.add("Standard3", cellname(9 + "col" + i), ByteBufferUtil.EMPTY_BYTE_BUFFER, System.currentTimeMillis()); + new RowUpdateBuilder(store.metadata, System.currentTimeMillis(), key) + .clustering(9 + "col" + i) + .add("val", ByteBufferUtil.EMPTY_BYTE_BUFFER) + .build() + .applyUnsafe(); } - rm.applyUnsafe(); store.forceBlockingFlush(); store.forceMajorCompaction(); assertEquals(1, store.getSSTables().size()); for (SSTableReader sstable : store.getSSTables()) { - assertEquals(ByteBufferUtil.string(sstable.getSSTableMetadata().minColumnNames.get(0)), "0col100"); - assertEquals(ByteBufferUtil.string(sstable.getSSTableMetadata().maxColumnNames.get(0)), "9col298"); + assertEquals(ByteBufferUtil.string(sstable.getSSTableMetadata().minClusteringValues.get(0)), "0col100"); + assertEquals(ByteBufferUtil.string(sstable.getSSTableMetadata().maxClusteringValues.get(0)), "9col298"); } } @@ -254,39 +274,38 @@ public class SSTableMetadataTest ColumnFamilyStore cfs = keyspace.getColumnFamilyStore("StandardComposite2"); - CellNameType type = cfs.getComparator(); - - ByteBuffer key = ByteBufferUtil.bytes("k"); for (int i = 0; i < 10; i++) { - Mutation rm = new Mutation(KEYSPACE1, key); - CellName colName = type.makeCellName(ByteBufferUtil.bytes("a"+(9-i)), ByteBufferUtil.bytes(i)); - rm.add("StandardComposite2", colName, ByteBufferUtil.EMPTY_BYTE_BUFFER, 0); - rm.applyUnsafe(); + new RowUpdateBuilder(cfs.metadata, 0, "k") + .clustering("a" + (9 - i), getBytes(i)) + .add("val", ByteBufferUtil.EMPTY_BYTE_BUFFER) + .build() + .applyUnsafe(); + } cfs.forceBlockingFlush(); - key = ByteBufferUtil.bytes("k2"); for (int i = 0; i < 10; i++) { - Mutation rm = new Mutation(KEYSPACE1, key); - CellName colName = type.makeCellName(ByteBufferUtil.bytes("b"+(9-i)), ByteBufferUtil.bytes(i)); - rm.add("StandardComposite2", colName, ByteBufferUtil.EMPTY_BYTE_BUFFER, 0); - rm.applyUnsafe(); + new RowUpdateBuilder(cfs.metadata, 0, "k2") + .clustering("b" + (9 - i), getBytes(i)) + .add("val", ByteBufferUtil.EMPTY_BYTE_BUFFER) + .build() + .applyUnsafe(); } cfs.forceBlockingFlush(); cfs.forceMajorCompaction(); assertEquals(cfs.getSSTables().size(), 1); for (SSTableReader sstable : cfs.getSSTables()) { - assertEquals("b9", ByteBufferUtil.string(sstable.getSSTableMetadata().maxColumnNames.get(0))); - assertEquals(9, ByteBufferUtil.toInt(sstable.getSSTableMetadata().maxColumnNames.get(1))); - assertEquals("a0", ByteBufferUtil.string(sstable.getSSTableMetadata().minColumnNames.get(0))); - assertEquals(0, ByteBufferUtil.toInt(sstable.getSSTableMetadata().minColumnNames.get(1))); + assertEquals("b9", ByteBufferUtil.string(sstable.getSSTableMetadata().maxClusteringValues.get(0))); + assertEquals(9, ByteBufferUtil.toInt(sstable.getSSTableMetadata().maxClusteringValues.get(1))); + assertEquals("a0", ByteBufferUtil.string(sstable.getSSTableMetadata().minClusteringValues.get(0))); + assertEquals(0, ByteBufferUtil.toInt(sstable.getSSTableMetadata().minClusteringValues.get(1))); } } - @Test + /*@Test public void testLegacyCounterShardTracking() { ColumnFamilyStore cfs = Keyspace.open(KEYSPACE1).getColumnFamilyStore("Counter1"); @@ -296,6 +315,7 @@ public class SSTableMetadataTest state.writeGlobal(CounterId.fromInt(1), 1L, 1L); state.writeLocal(CounterId.fromInt(2), 1L, 1L); state.writeRemote(CounterId.fromInt(3), 1L, 1L); + ColumnFamily cells = ArrayBackedSortedColumns.factory.create(cfs.metadata); cells.addColumn(new BufferCounterCell(cellname("col"), state.context, 1L, Long.MIN_VALUE)); new Mutation(Util.dk("k").getKey(), cells).applyUnsafe(); @@ -334,5 +354,5 @@ public class SSTableMetadataTest cfs.forceBlockingFlush(); assertFalse(cfs.getSSTables().iterator().next().getSSTableMetadata().hasLegacyCounterShards); cfs.truncateBlocking(); - } + } */ } diff --git a/test/unit/org/apache/cassandra/io/sstable/SSTableReaderTest.java b/test/unit/org/apache/cassandra/io/sstable/SSTableReaderTest.java index 682d999f90..3b8b90c0cf 100644 --- a/test/unit/org/apache/cassandra/io/sstable/SSTableReaderTest.java +++ b/test/unit/org/apache/cassandra/io/sstable/SSTableReaderTest.java @@ -1,6 +1,4 @@ -package org.apache.cassandra.io.sstable; /* - * * Licensed to the Apache Software Foundation (ASF) under one * or more contributor license agreements. See the NOTICE file * distributed with this work for additional information @@ -9,17 +7,17 @@ package org.apache.cassandra.io.sstable; * "License"); you may not use this file except in compliance * with the License. You may obtain a copy of the License at * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. + * http://www.apache.org/licenses/LICENSE-2.0 * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. */ +package org.apache.cassandra.io.sstable; + import java.io.IOException; import java.nio.ByteBuffer; @@ -34,10 +32,6 @@ import java.util.concurrent.ScheduledThreadPoolExecutor; import java.util.concurrent.ThreadPoolExecutor; import com.google.common.collect.Sets; -import org.apache.cassandra.cache.CachingOptions; -import org.apache.cassandra.config.KSMetaData; -import org.apache.cassandra.io.sstable.format.SSTableReader; -import org.apache.cassandra.locator.SimpleStrategy; import org.junit.Assert; import org.junit.BeforeClass; import org.junit.Test; @@ -46,37 +40,42 @@ import org.junit.runner.RunWith; import org.apache.cassandra.OrderedJUnit4ClassRunner; import org.apache.cassandra.SchemaLoader; import org.apache.cassandra.Util; +import org.apache.cassandra.cache.CachingOptions; import org.apache.cassandra.config.DatabaseDescriptor; +import org.apache.cassandra.config.KSMetaData; import org.apache.cassandra.cql3.Operator; import org.apache.cassandra.db.BufferDecoratedKey; -import org.apache.cassandra.db.ColumnFamily; import org.apache.cassandra.db.ColumnFamilyStore; import org.apache.cassandra.db.DecoratedKey; -import org.apache.cassandra.db.IndexExpression; import org.apache.cassandra.db.Keyspace; -import org.apache.cassandra.db.Mutation; -import org.apache.cassandra.db.Row; -import org.apache.cassandra.db.RowPosition; -import org.apache.cassandra.db.columniterator.IdentityQueryFilter; +import org.apache.cassandra.db.ReadCommand; +import org.apache.cassandra.db.RowUpdateBuilder; import org.apache.cassandra.db.compaction.CompactionManager; import org.apache.cassandra.db.compaction.OperationType; -import org.apache.cassandra.db.composites.Composites; import org.apache.cassandra.db.lifecycle.LifecycleTransaction; +import org.apache.cassandra.db.index.SecondaryIndexSearcher; +import org.apache.cassandra.db.rows.Row; +import org.apache.cassandra.db.partitions.ArrayBackedPartition; +import org.apache.cassandra.db.partitions.UnfilteredPartitionIterator; import org.apache.cassandra.dht.LocalPartitioner; import org.apache.cassandra.dht.LocalPartitioner.LocalToken; import org.apache.cassandra.dht.Range; import org.apache.cassandra.dht.Token; +import org.apache.cassandra.io.sstable.format.SSTableReader; import org.apache.cassandra.io.util.FileDataInput; import org.apache.cassandra.io.util.MmappedSegmentedFile; import org.apache.cassandra.io.util.SegmentedFile; +import org.apache.cassandra.locator.SimpleStrategy; import org.apache.cassandra.service.CacheService; import org.apache.cassandra.service.StorageService; import org.apache.cassandra.utils.ByteBufferUtil; +import org.apache.cassandra.utils.FBUtilities; import org.apache.cassandra.utils.Pair; -import static org.apache.cassandra.Util.cellname; + import static org.junit.Assert.assertEquals; import static org.junit.Assert.assertFalse; import static org.junit.Assert.assertTrue; +import static org.apache.cassandra.utils.ByteBufferUtil.bytes; @RunWith(OrderedJUnit4ClassRunner.class) public class SSTableReaderTest @@ -101,7 +100,7 @@ public class SSTableReaderTest KSMetaData.optsWithRF(1), SchemaLoader.standardCFMD(KEYSPACE1, CF_STANDARD), SchemaLoader.standardCFMD(KEYSPACE1, CF_STANDARD2), - SchemaLoader.indexCFMD(KEYSPACE1, CF_INDEXED, true), + SchemaLoader.compositeIndexCFMD(KEYSPACE1, CF_INDEXED, true), SchemaLoader.standardCFMD(KEYSPACE1, CF_STANDARDLOWINDEXINTERVAL) .minIndexInterval(8) .maxIndexInterval(256) @@ -118,23 +117,24 @@ public class SSTableReaderTest CompactionManager.instance.disableAutoCompaction(); for (int j = 0; j < 10; j++) { - ByteBuffer key = ByteBufferUtil.bytes(String.valueOf(j)); - Mutation rm = new Mutation(KEYSPACE1, key); - rm.add("Standard2", cellname("0"), ByteBufferUtil.EMPTY_BYTE_BUFFER, j); - rm.applyUnsafe(); + new RowUpdateBuilder(store.metadata, j, String.valueOf(j)) + .clustering("0") + .add("val", ByteBufferUtil.EMPTY_BYTE_BUFFER) + .build() + .applyUnsafe(); } store.forceBlockingFlush(); CompactionManager.instance.performMaximal(store, false); List> ranges = new ArrayList>(); // 1 key - ranges.add(new Range(t(0), t(1))); + ranges.add(new Range<>(t(0), t(1))); // 2 keys - ranges.add(new Range(t(2), t(4))); + ranges.add(new Range<>(t(2), t(4))); // wrapping range from key to end - ranges.add(new Range(t(6), StorageService.getPartitioner().getMinimumToken())); + ranges.add(new Range<>(t(6), StorageService.getPartitioner().getMinimumToken())); // empty range (should be ignored) - ranges.add(new Range(t(9), t(91))); + ranges.add(new Range<>(t(9), t(91))); // confirm that positions increase continuously SSTableReader sstable = store.getSSTables().iterator().next(); @@ -159,10 +159,11 @@ public class SSTableReaderTest CompactionManager.instance.disableAutoCompaction(); for (int j = 0; j < 100; j += 2) { - ByteBuffer key = ByteBufferUtil.bytes(String.valueOf(j)); - Mutation rm = new Mutation(KEYSPACE1, key); - rm.add("Standard1", cellname("0"), ByteBufferUtil.EMPTY_BYTE_BUFFER, j); - rm.applyUnsafe(); + new RowUpdateBuilder(store.metadata, j, String.valueOf(j)) + .clustering("0") + .add("val", ByteBufferUtil.EMPTY_BYTE_BUFFER) + .build() + .applyUnsafe(); } store.forceBlockingFlush(); CompactionManager.instance.performMaximal(store, false); @@ -194,10 +195,11 @@ public class SSTableReaderTest for (int j = 0; j < 100; j += 2) { - ByteBuffer key = ByteBufferUtil.bytes(String.valueOf(j)); - Mutation rm = new Mutation(KEYSPACE1, key); - rm.add("Standard1", cellname("0"), ByteBufferUtil.EMPTY_BYTE_BUFFER, j); - rm.applyUnsafe(); + new RowUpdateBuilder(store.metadata, j, String.valueOf(j)) + .clustering("0") + .add("val", ByteBufferUtil.EMPTY_BYTE_BUFFER) + .build() + .applyUnsafe(); } store.forceBlockingFlush(); @@ -220,23 +222,24 @@ public class SSTableReaderTest for (int j = 0; j < 10; j++) { - ByteBuffer key = ByteBufferUtil.bytes(String.valueOf(j)); - Mutation rm = new Mutation(KEYSPACE1, key); - rm.add("Standard1", cellname("0"), ByteBufferUtil.EMPTY_BYTE_BUFFER, j); - rm.apply(); + new RowUpdateBuilder(store.metadata, j, String.valueOf(j)) + .clustering("0") + .add("val", ByteBufferUtil.EMPTY_BYTE_BUFFER) + .build() + .applyUnsafe(); } + store.forceBlockingFlush(); SSTableReader sstable = store.getSSTables().iterator().next(); assertEquals(0, sstable.getReadMeter().count()); DecoratedKey key = sstable.partitioner.decorateKey(ByteBufferUtil.bytes("4")); - store.getColumnFamily(key, Composites.EMPTY, Composites.EMPTY, false, 100, 100); + Util.getAll(Util.cmd(store, key).build()); assertEquals(1, sstable.getReadMeter().count()); - store.getColumnFamily(key, cellname("0"), cellname("0"), false, 100, 100); + + Util.getAll(Util.cmd(store, key).includeRow("0").build()); assertEquals(2, sstable.getReadMeter().count()); - store.getColumnFamily(Util.namesQueryFilter(store, key, cellname("0"))); - assertEquals(3, sstable.getReadMeter().count()); } @Test @@ -250,10 +253,13 @@ public class SSTableReaderTest CompactionManager.instance.disableAutoCompaction(); for (int j = 0; j < 10; j++) { - ByteBuffer key = ByteBufferUtil.bytes(String.valueOf(j)); - Mutation rm = new Mutation(KEYSPACE1, key); - rm.add("Standard2", cellname("0"), ByteBufferUtil.EMPTY_BYTE_BUFFER, j); - rm.applyUnsafe(); + + new RowUpdateBuilder(store.metadata, j, String.valueOf(j)) + .clustering("0") + .add("val", ByteBufferUtil.EMPTY_BYTE_BUFFER) + .build() + .applyUnsafe(); + } store.forceBlockingFlush(); CompactionManager.instance.performMaximal(store, false); @@ -278,11 +284,14 @@ public class SSTableReaderTest { // Create secondary index and flush to disk Keyspace keyspace = Keyspace.open(KEYSPACE1); - ColumnFamilyStore store = keyspace.getColumnFamilyStore("Indexed1"); - ByteBuffer key = ByteBufferUtil.bytes(String.valueOf("k1")); - Mutation rm = new Mutation(KEYSPACE1, key); - rm.add("Indexed1", cellname("birthdate"), ByteBufferUtil.bytes(1L), System.currentTimeMillis()); - rm.applyUnsafe(); + ColumnFamilyStore store = keyspace.getColumnFamilyStore(CF_INDEXED); + + new RowUpdateBuilder(store.metadata, System.currentTimeMillis(), "k1") + .clustering("0") + .add("birthdate", 1L) + .build() + .applyUnsafe(); + store.forceBlockingFlush(); // check if opening and querying works @@ -298,10 +307,11 @@ public class SSTableReaderTest CompactionManager.instance.disableAutoCompaction(); for (int j = 0; j < 10; j++) { - ByteBuffer key = ByteBufferUtil.bytes(String.valueOf(j)); - Mutation rm = new Mutation("Keyspace1", key); - rm.add("Standard2", cellname("0"), ByteBufferUtil.EMPTY_BYTE_BUFFER, j); - rm.apply(); + new RowUpdateBuilder(store.metadata, j, String.valueOf(j)) + .clustering("0") + .add("val", ByteBufferUtil.EMPTY_BYTE_BUFFER) + .build() + .applyUnsafe(); } store.forceBlockingFlush(); CompactionManager.instance.performMaximal(store, false); @@ -343,10 +353,14 @@ public class SSTableReaderTest lastKey = key; if (store.metadata.getKeyValidator().compare(lastKey.getKey(), key.getKey()) < 0) lastKey = key; - Mutation rm = new Mutation(ks, key.getKey()); - rm.add(cf, cellname("col"), - ByteBufferUtil.EMPTY_BYTE_BUFFER, timestamp); - rm.applyUnsafe(); + + + new RowUpdateBuilder(store.metadata, timestamp, key.getKey()) + .clustering("col") + .add("val", ByteBufferUtil.EMPTY_BYTE_BUFFER) + .build() + .applyUnsafe(); + } store.forceBlockingFlush(); @@ -367,13 +381,16 @@ public class SSTableReaderTest { Keyspace keyspace = Keyspace.open(KEYSPACE1); ColumnFamilyStore store = keyspace.getColumnFamilyStore("Indexed1"); - ByteBuffer key = ByteBufferUtil.bytes(String.valueOf("k1")); - Mutation rm = new Mutation(KEYSPACE1, key); - rm.add("Indexed1", cellname("birthdate"), ByteBufferUtil.bytes(1L), System.currentTimeMillis()); - rm.applyUnsafe(); + + new RowUpdateBuilder(store.metadata, System.currentTimeMillis(), "k1") + .clustering("0") + .add("birthdate", 1L) + .build() + .applyUnsafe(); + store.forceBlockingFlush(); - ColumnFamilyStore indexCfs = store.indexManager.getIndexForColumn(ByteBufferUtil.bytes("birthdate")).getIndexCfs(); + ColumnFamilyStore indexCfs = store.indexManager.getIndexForColumn(store.metadata.getColumnDefinition(bytes("birthdate"))).getIndexCfs(); assert indexCfs.partitioner instanceof LocalPartitioner; SSTableReader sstable = indexCfs.getSSTables().iterator().next(); assert sstable.first.getToken() instanceof LocalToken; @@ -394,10 +411,13 @@ public class SSTableReaderTest { Keyspace keyspace = Keyspace.open(KEYSPACE1); ColumnFamilyStore store = keyspace.getColumnFamilyStore("Standard1"); - ByteBuffer key = ByteBufferUtil.bytes(String.valueOf("k1")); - Mutation rm = new Mutation(KEYSPACE1, key); - rm.add("Standard1", cellname("xyz"), ByteBufferUtil.bytes("abc"), 0); - rm.applyUnsafe(); + + new RowUpdateBuilder(store.metadata, 0, "k1") + .clustering("xyz") + .add("val", "abc") + .build() + .applyUnsafe(); + store.forceBlockingFlush(); boolean foundScanner = false; for (SSTableReader s : store.getSSTables()) @@ -423,10 +443,13 @@ public class SSTableReaderTest CompactionManager.instance.disableAutoCompaction(); for (int j = 0; j < 130; j++) { - ByteBuffer key = ByteBufferUtil.bytes(String.valueOf(j)); - Mutation rm = new Mutation(KEYSPACE1, key); - rm.add("Standard2", cellname("0"), ByteBufferUtil.EMPTY_BYTE_BUFFER, j); - rm.applyUnsafe(); + + new RowUpdateBuilder(store.metadata, j, String.valueOf(j)) + .clustering("0") + .add("val", ByteBufferUtil.EMPTY_BYTE_BUFFER) + .build() + .applyUnsafe(); + } store.forceBlockingFlush(); CompactionManager.instance.performMaximal(store, false); @@ -457,13 +480,15 @@ public class SSTableReaderTest final ColumnFamilyStore store = keyspace.getColumnFamilyStore("StandardLowIndexInterval"); // index interval of 8, no key caching CompactionManager.instance.disableAutoCompaction(); - final int NUM_ROWS = 512; - for (int j = 0; j < NUM_ROWS; j++) + final int NUM_PARTITIONS = 512; + for (int j = 0; j < NUM_PARTITIONS; j++) { - ByteBuffer key = ByteBufferUtil.bytes(String.format("%3d", j)); - Mutation rm = new Mutation(KEYSPACE1, key); - rm.add("StandardLowIndexInterval", Util.cellname("0"), ByteBufferUtil.bytes(String.format("%3d", j)), j); - rm.applyUnsafe(); + new RowUpdateBuilder(store.metadata, j, String.format("%3d", j)) + .clustering("0") + .add("val", String.format("%3d", j)) + .build() + .applyUnsafe(); + } store.forceBlockingFlush(); CompactionManager.instance.performMaximal(store, false); @@ -473,8 +498,8 @@ public class SSTableReaderTest final SSTableReader sstable = sstables.iterator().next(); ThreadPoolExecutor executor = new ScheduledThreadPoolExecutor(5); - List futures = new ArrayList<>(NUM_ROWS * 2); - for (int i = 0; i < NUM_ROWS; i++) + List futures = new ArrayList<>(NUM_PARTITIONS * 2); + for (int i = 0; i < NUM_PARTITIONS; i++) { final ByteBuffer key = ByteBufferUtil.bytes(String.format("%3d", i)); final int index = i; @@ -483,9 +508,8 @@ public class SSTableReaderTest { public void run() { - ColumnFamily result = store.getColumnFamily(sstable.partitioner.decorateKey(key), Composites.EMPTY, Composites.EMPTY, false, 100, 100); - assertFalse(result.isEmpty()); - assertEquals(0, ByteBufferUtil.compare(String.format("%3d", index).getBytes(), result.getColumn(Util.cellname("0")).value())); + Row row = Util.getOnlyRowUnfiltered(Util.cmd(store, key).build()); + assertEquals(0, ByteBufferUtil.compare(String.format("%3d", index).getBytes(), row.iterator().next().value())); } })); @@ -521,12 +545,16 @@ public class SSTableReaderTest for (ColumnFamilyStore cfs : indexedCFS.concatWithIndexes()) clearAndLoad(cfs); + ByteBuffer bBB = bytes("birthdate"); + // query using index to see if sstable for secondary index opens - IndexExpression expr = new IndexExpression(ByteBufferUtil.bytes("birthdate"), Operator.EQ, ByteBufferUtil.bytes(1L)); - List clause = Arrays.asList(expr); - Range range = Util.range("", ""); - List rows = indexedCFS.search(range, clause, new IdentityQueryFilter(), 100); - assert rows.size() == 1; + ReadCommand rc = Util.cmd(indexedCFS).fromKeyIncl("k1").toKeyIncl("k3") + .columns("birthdate") + .filterOn("birthdate", Operator.EQ, 1L) + .build(); + List searchers = indexedCFS.indexManager.getIndexSearchersFor(rc); + assertEquals(searchers.size(), 1); + assertEquals(1, Util.getAll(rc).size()); } private List> makeRanges(Token left, Token right) diff --git a/test/unit/org/apache/cassandra/io/sstable/SSTableRewriterTest.java b/test/unit/org/apache/cassandra/io/sstable/SSTableRewriterTest.java index 8f6ec168d1..9e4c93910c 100644 --- a/test/unit/org/apache/cassandra/io/sstable/SSTableRewriterTest.java +++ b/test/unit/org/apache/cassandra/io/sstable/SSTableRewriterTest.java @@ -15,6 +15,7 @@ * See the License for the specific language governing permissions and * limitations under the License. */ + package org.apache.cassandra.io.sstable; import java.io.File; @@ -33,27 +34,39 @@ import org.junit.Test; import org.apache.cassandra.SchemaLoader; import org.apache.cassandra.Util; +import org.apache.cassandra.UpdateBuilder; import org.apache.cassandra.config.KSMetaData; -import org.apache.cassandra.db.*; -import org.apache.cassandra.db.compaction.AbstractCompactedRow; +import org.apache.cassandra.db.ColumnFamilyStore; +import org.apache.cassandra.db.DecoratedKey; +import org.apache.cassandra.db.Keyspace; +import org.apache.cassandra.db.RowUpdateBuilder; +import org.apache.cassandra.db.SerializationHeader; +import org.apache.cassandra.db.rows.RowStats; +import org.apache.cassandra.db.rows.UnfilteredRowIterator; import org.apache.cassandra.db.compaction.AbstractCompactionStrategy; import org.apache.cassandra.db.compaction.CompactionController; -import org.apache.cassandra.db.compaction.LazilyCompactedRow; +import org.apache.cassandra.db.compaction.CompactionIterator; import org.apache.cassandra.db.compaction.OperationType; +import org.apache.cassandra.db.compaction.SSTableSplitter; +import org.apache.cassandra.db.partitions.ArrayBackedPartition; +import org.apache.cassandra.db.partitions.PartitionUpdate; +import org.apache.cassandra.dht.Range; +import org.apache.cassandra.dht.Token; import org.apache.cassandra.exceptions.ConfigurationException; -import org.apache.cassandra.io.util.FileUtils; +import org.apache.cassandra.io.sstable.format.SSTableFormat; import org.apache.cassandra.io.sstable.format.SSTableReader; import org.apache.cassandra.io.sstable.format.SSTableWriter; +import org.apache.cassandra.io.util.FileUtils; import org.apache.cassandra.locator.SimpleStrategy; import org.apache.cassandra.db.compaction.SSTableSplitter; import org.apache.cassandra.db.lifecycle.LifecycleTransaction; import org.apache.cassandra.dht.Range; import org.apache.cassandra.dht.Token; import org.apache.cassandra.metrics.StorageMetrics; -import org.apache.cassandra.service.StorageService; import org.apache.cassandra.utils.ByteBufferUtil; import org.apache.cassandra.utils.FBUtilities; import org.apache.cassandra.utils.Pair; +import org.apache.cassandra.utils.UUIDGen; import static org.junit.Assert.*; @@ -67,9 +80,9 @@ public class SSTableRewriterTest extends SchemaLoader { SchemaLoader.prepareServer(); SchemaLoader.createKeyspace(KEYSPACE, - SimpleStrategy.class, - KSMetaData.optsWithRF(1), - SchemaLoader.standardCFMD(KEYSPACE, CF)); + SimpleStrategy.class, + KSMetaData.optsWithRF(1), + SchemaLoader.standardCFMD(KEYSPACE, CF)); } @After @@ -91,26 +104,27 @@ public class SSTableRewriterTest extends SchemaLoader for (int j = 0; j < 100; j ++) { - ByteBuffer key = ByteBufferUtil.bytes(String.valueOf(j)); - Mutation rm = new Mutation(KEYSPACE, key); - rm.add(CF, Util.cellname("0"), ByteBufferUtil.EMPTY_BYTE_BUFFER, j); - rm.apply(); + new RowUpdateBuilder(cfs.metadata, j, String.valueOf(j)) + .clustering("0") + .add("val", ByteBufferUtil.EMPTY_BYTE_BUFFER) + .build() + .apply(); } cfs.forceBlockingFlush(); Set sstables = new HashSet<>(cfs.getSSTables()); assertEquals(1, sstables.size()); assertEquals(sstables.iterator().next().bytesOnDisk(), cfs.metric.liveDiskSpaceUsed.getCount()); + int nowInSec = FBUtilities.nowInSeconds(); try (AbstractCompactionStrategy.ScannerList scanners = cfs.getCompactionStrategyManager().getScanners(sstables); LifecycleTransaction txn = cfs.getTracker().tryModify(sstables, OperationType.UNKNOWN); - SSTableRewriter writer = new SSTableRewriter(cfs, txn, 1000, false);) + SSTableRewriter writer = new SSTableRewriter(cfs, txn, 1000, false); + CompactionController controller = new CompactionController(cfs, sstables, cfs.gcBefore(nowInSec)); + CompactionIterator ci = new CompactionIterator(OperationType.COMPACTION, scanners.scanners, controller, nowInSec, UUIDGen.getTimeUUID())) { - ISSTableScanner scanner = scanners.scanners.get(0); - CompactionController controller = new CompactionController(cfs, sstables, cfs.gcBefore(System.currentTimeMillis())); writer.switchWriter(getWriter(cfs, sstables.iterator().next().descriptor.directory)); - while(scanner.hasNext()) + while(ci.hasNext()) { - AbstractCompactedRow row = new LazilyCompactedRow(controller, Arrays.asList(scanner.next())); - writer.append(row); + writer.append(ci.next()); } writer.finish(); } @@ -132,17 +146,17 @@ public class SSTableRewriterTest extends SchemaLoader Set sstables = new HashSet<>(cfs.getSSTables()); assertEquals(1, sstables.size()); + int nowInSec = FBUtilities.nowInSeconds(); try (AbstractCompactionStrategy.ScannerList scanners = cfs.getCompactionStrategyManager().getScanners(sstables); LifecycleTransaction txn = cfs.getTracker().tryModify(sstables, OperationType.UNKNOWN); - SSTableRewriter writer = new SSTableRewriter(cfs, txn, 1000, false, 10000000);) + SSTableRewriter writer = new SSTableRewriter(cfs, txn, 1000, false, 10000000); + CompactionController controller = new CompactionController(cfs, sstables, cfs.gcBefore(nowInSec)); + CompactionIterator ci = new CompactionIterator(OperationType.COMPACTION, scanners.scanners, controller, nowInSec, UUIDGen.getTimeUUID())) { - ISSTableScanner scanner = scanners.scanners.get(0); - CompactionController controller = new CompactionController(cfs, sstables, cfs.gcBefore(System.currentTimeMillis())); writer.switchWriter(getWriter(cfs, sstables.iterator().next().descriptor.directory)); - while (scanner.hasNext()) + while (ci.hasNext()) { - AbstractCompactedRow row = new LazilyCompactedRow(controller, Arrays.asList(scanner.next())); - writer.append(row); + writer.append(ci.next()); } writer.finish(); } @@ -166,19 +180,20 @@ public class SSTableRewriterTest extends SchemaLoader Set sstables = new HashSet<>(cfs.getSSTables()); assertEquals(1, sstables.size()); + int nowInSec = FBUtilities.nowInSeconds(); boolean checked = false; try (AbstractCompactionStrategy.ScannerList scanners = cfs.getCompactionStrategyManager().getScanners(sstables); LifecycleTransaction txn = cfs.getTracker().tryModify(sstables, OperationType.UNKNOWN); - SSTableRewriter writer = new SSTableRewriter(cfs, txn, 1000, false, 10000000)) + SSTableRewriter writer = new SSTableRewriter(cfs, txn, 1000, false, 10000000); + CompactionController controller = new CompactionController(cfs, sstables, cfs.gcBefore(nowInSec)); + CompactionIterator ci = new CompactionIterator(OperationType.COMPACTION, scanners.scanners, controller, nowInSec, UUIDGen.getTimeUUID())) { - ISSTableScanner scanner = scanners.scanners.get(0); - CompactionController controller = new CompactionController(cfs, sstables, cfs.gcBefore(System.currentTimeMillis())); writer.switchWriter(getWriter(cfs, sstables.iterator().next().descriptor.directory)); - while (scanner.hasNext()) + while (ci.hasNext()) { - AbstractCompactedRow row = new LazilyCompactedRow(controller, Arrays.asList(scanner.next())); + UnfilteredRowIterator row = ci.next(); writer.append(row); - if (!checked && writer.currentWriter().getFilePointer() > 15000000) + if (!checked && writer.currentWriter().getFilePointer() > 1500000) { checked = true; for (SSTableReader sstable : cfs.getSSTables()) @@ -218,20 +233,42 @@ public class SSTableRewriterTest extends SchemaLoader ColumnFamilyStore cfs = keyspace.getColumnFamilyStore(CF); truncate(cfs); assertEquals(0, cfs.metric.liveDiskSpaceUsed.getCount()); - ArrayBackedSortedColumns cf = ArrayBackedSortedColumns.factory.create(cfs.metadata); - for (int i = 0; i < 100; i++) - cf.addColumn(Util.cellname(i), ByteBuffer.allocate(1000), 1); - File dir = cfs.directories.getDirectoryForNewSSTables(); - try (SSTableWriter writer = getWriter(cfs, dir);) + File dir = cfs.directories.getDirectoryForNewSSTables(); + try (SSTableWriter writer = getWriter(cfs, dir)) { for (int i = 0; i < 10000; i++) - writer.append(StorageService.getPartitioner().decorateKey(random(i, 10)), cf); + { + RowUpdateBuilder builder = new RowUpdateBuilder(cfs.metadata, 1, random(i, 10)); + + PartitionUpdate update = null; + + for (int j = 0; j < 100; j++) + { + builder.clustering("" + j).add("val", ByteBuffer.allocate(1000)); + update = builder.buildUpdate(); + } + + writer.append(update.unfilteredIterator()); + } + SSTableReader s = writer.setMaxDataAge(1000).openEarly(); assert s != null; assertFileCounts(dir.list(), 2, 2); for (int i = 10000; i < 20000; i++) - writer.append(StorageService.getPartitioner().decorateKey(random(i, 10)), cf); + { + RowUpdateBuilder builder = new RowUpdateBuilder(cfs.metadata, 1, random(i, 10)); + + PartitionUpdate update = null; + + for (int j = 0; j < 100; j++) + { + builder.clustering("" + j).add("val", ByteBuffer.allocate(1000)); + update = builder.buildUpdate(); + } + + writer.append(update.unfilteredIterator()); + } SSTableReader s2 = writer.setMaxDataAge(1000).openEarly(); assertTrue(s.last.compareTo(s2.last) < 0); assertFileCounts(dir.list(), 2, 2); @@ -271,12 +308,14 @@ public class SSTableRewriterTest extends SchemaLoader try (ISSTableScanner scanner = s.getScanner(); CompactionController controller = new CompactionController(cfs, compacting, 0); LifecycleTransaction txn = cfs.getTracker().tryModify(compacting, OperationType.UNKNOWN); - SSTableRewriter rewriter = new SSTableRewriter(cfs, txn, 1000, false, 10000000)) + SSTableRewriter rewriter = new SSTableRewriter(cfs, txn, 1000, false, 10000000); + CompactionIterator ci = new CompactionIterator(OperationType.COMPACTION, Collections.singletonList(scanner), controller, FBUtilities.nowInSeconds(), UUIDGen.getTimeUUID())) { rewriter.switchWriter(getWriter(cfs, s.descriptor.directory)); - while(scanner.hasNext()) + + while(ci.hasNext()) { - rewriter.append(new LazilyCompactedRow(controller, Arrays.asList(scanner.next()))); + rewriter.append(ci.next()); if (rewriter.currentWriter().getOnDiskFilePointer() > 25000000) { rewriter.switchWriter(getWriter(cfs, s.descriptor.directory)); @@ -321,12 +360,14 @@ public class SSTableRewriterTest extends SchemaLoader try (ISSTableScanner scanner = s.getScanner(); CompactionController controller = new CompactionController(cfs, compacting, 0); LifecycleTransaction txn = cfs.getTracker().tryModify(compacting, OperationType.UNKNOWN); - SSTableRewriter rewriter = new SSTableRewriter(cfs, txn, 1000, false, 10000000)) + SSTableRewriter rewriter = new SSTableRewriter(cfs, txn, 1000, false, 10000000); + CompactionIterator ci = new CompactionIterator(OperationType.COMPACTION, Collections.singletonList(scanner), controller, FBUtilities.nowInSeconds(), UUIDGen.getTimeUUID())) { rewriter.switchWriter(getWriter(cfs, s.descriptor.directory)); - while(scanner.hasNext()) + + while(ci.hasNext()) { - rewriter.append(new LazilyCompactedRow(controller, Arrays.asList(scanner.next()))); + rewriter.append(ci.next()); if (rewriter.currentWriter().getOnDiskFilePointer() > 25000000) { rewriter.switchWriter(getWriter(cfs, s.descriptor.directory)); @@ -353,18 +394,21 @@ public class SSTableRewriterTest extends SchemaLoader { public void run(ISSTableScanner scanner, CompactionController controller, SSTableReader sstable, ColumnFamilyStore cfs, SSTableRewriter rewriter) { - int files = 1; - while(scanner.hasNext()) + try (CompactionIterator ci = new CompactionIterator(OperationType.COMPACTION, Collections.singletonList(scanner), controller, FBUtilities.nowInSeconds(), UUIDGen.getTimeUUID())) { - rewriter.append(new LazilyCompactedRow(controller, Arrays.asList(scanner.next()))); - if (rewriter.currentWriter().getFilePointer() > 25000000) + int files = 1; + while (ci.hasNext()) { - rewriter.switchWriter(getWriter(cfs, sstable.descriptor.directory)); - files++; - assertEquals(cfs.getSSTables().size(), files); // we have one original file plus the ones we have switched out. + rewriter.append(ci.next()); + if (rewriter.currentWriter().getFilePointer() > 25000000) + { + rewriter.switchWriter(getWriter(cfs, sstable.descriptor.directory)); + files++; + assertEquals(cfs.getSSTables().size(), files); // we have one original file plus the ones we have switched out. + } } + rewriter.abort(); } - rewriter.abort(); } }); } @@ -376,21 +420,24 @@ public class SSTableRewriterTest extends SchemaLoader { public void run(ISSTableScanner scanner, CompactionController controller, SSTableReader sstable, ColumnFamilyStore cfs, SSTableRewriter rewriter) { - int files = 1; - while(scanner.hasNext()) + try (CompactionIterator ci = new CompactionIterator(OperationType.COMPACTION, Collections.singletonList(scanner), controller, FBUtilities.nowInSeconds(), UUIDGen.getTimeUUID())) { - rewriter.append(new LazilyCompactedRow(controller, Arrays.asList(scanner.next()))); - if (rewriter.currentWriter().getFilePointer() > 25000000) + int files = 1; + while (ci.hasNext()) { - rewriter.switchWriter(getWriter(cfs, sstable.descriptor.directory)); - files++; - assertEquals(cfs.getSSTables().size(), files); // we have one original file plus the ones we have switched out. - } - if (files == 3) - { - //testing to abort when we have nothing written in the new file - rewriter.abort(); - break; + rewriter.append(ci.next()); + if (rewriter.currentWriter().getFilePointer() > 25000000) + { + rewriter.switchWriter(getWriter(cfs, sstable.descriptor.directory)); + files++; + assertEquals(cfs.getSSTables().size(), files); // we have one original file plus the ones we have switched out. + } + if (files == 3) + { + //testing to abort when we have nothing written in the new file + rewriter.abort(); + break; + } } } } @@ -404,18 +451,21 @@ public class SSTableRewriterTest extends SchemaLoader { public void run(ISSTableScanner scanner, CompactionController controller, SSTableReader sstable, ColumnFamilyStore cfs, SSTableRewriter rewriter) { - int files = 1; - while(scanner.hasNext()) + try(CompactionIterator ci = new CompactionIterator(OperationType.COMPACTION, Collections.singletonList(scanner), controller, FBUtilities.nowInSeconds(), UUIDGen.getTimeUUID())) { - rewriter.append(new LazilyCompactedRow(controller, Arrays.asList(scanner.next()))); - if (files == 1 && rewriter.currentWriter().getFilePointer() > 10000000) + int files = 1; + while (ci.hasNext()) { - rewriter.switchWriter(getWriter(cfs, sstable.descriptor.directory)); - files++; - assertEquals(cfs.getSSTables().size(), files); // we have one original file plus the ones we have switched out. + rewriter.append(ci.next()); + if (files == 1 && rewriter.currentWriter().getFilePointer() > 10000000) + { + rewriter.switchWriter(getWriter(cfs, sstable.descriptor.directory)); + files++; + assertEquals(cfs.getSSTables().size(), files); // we have one original file plus the ones we have switched out. + } } + rewriter.abort(); } - rewriter.abort(); } }); } @@ -442,7 +492,7 @@ public class SSTableRewriterTest extends SchemaLoader try (ISSTableScanner scanner = s.getScanner(); CompactionController controller = new CompactionController(cfs, compacting, 0); LifecycleTransaction txn = cfs.getTracker().tryModify(compacting, OperationType.UNKNOWN); - SSTableRewriter rewriter = new SSTableRewriter(cfs, txn, 1000, false, 10000000);) + SSTableRewriter rewriter = new SSTableRewriter(cfs, txn, 1000, false, 10000000)) { rewriter.switchWriter(getWriter(cfs, s.descriptor.directory)); test.run(scanner, controller, s, cfs, rewriter); @@ -474,13 +524,14 @@ public class SSTableRewriterTest extends SchemaLoader try (ISSTableScanner scanner = s.getScanner(); CompactionController controller = new CompactionController(cfs, compacting, 0); LifecycleTransaction txn = cfs.getTracker().tryModify(compacting, OperationType.UNKNOWN); - SSTableRewriter rewriter = new SSTableRewriter(cfs, txn, 1000, false, 10000000)) + SSTableRewriter rewriter = new SSTableRewriter(cfs, txn, 1000, false, 10000000); + CompactionIterator ci = new CompactionIterator(OperationType.COMPACTION, Collections.singletonList(scanner), controller, FBUtilities.nowInSeconds(), UUIDGen.getTimeUUID())) { rewriter.switchWriter(getWriter(cfs, s.descriptor.directory)); - while(scanner.hasNext()) + while(ci.hasNext()) { - rewriter.append(new LazilyCompactedRow(controller, Arrays.asList(scanner.next()))); - if (rewriter.currentWriter().getFilePointer() > 25000000) + rewriter.append(ci.next()); + if (rewriter.currentWriter().getFilePointer() > 2500000) { rewriter.switchWriter(getWriter(cfs, s.descriptor.directory)); files++; @@ -519,12 +570,13 @@ public class SSTableRewriterTest extends SchemaLoader try (ISSTableScanner scanner = s.getScanner(); CompactionController controller = new CompactionController(cfs, compacting, 0); LifecycleTransaction txn = cfs.getTracker().tryModify(compacting, OperationType.UNKNOWN); - SSTableRewriter rewriter = new SSTableRewriter(cfs, txn, 1000, false, 10000000)) + SSTableRewriter rewriter = new SSTableRewriter(cfs, txn, 1000, false, 10000000); + CompactionIterator ci = new CompactionIterator(OperationType.COMPACTION, Collections.singletonList(scanner), controller, FBUtilities.nowInSeconds(), UUIDGen.getTimeUUID())) { rewriter.switchWriter(getWriter(cfs, s.descriptor.directory)); - while(scanner.hasNext()) + while(ci.hasNext()) { - rewriter.append(new LazilyCompactedRow(controller, Arrays.asList(scanner.next()))); + rewriter.append(ci.next()); if (rewriter.currentWriter().getOnDiskFilePointer() > 25000000) { rewriter.switchWriter(getWriter(cfs, s.descriptor.directory)); @@ -560,12 +612,13 @@ public class SSTableRewriterTest extends SchemaLoader try (ISSTableScanner scanner = s.getScanner(); CompactionController controller = new CompactionController(cfs, compacting, 0); LifecycleTransaction txn = cfs.getTracker().tryModify(compacting, OperationType.UNKNOWN); - SSTableRewriter rewriter = new SSTableRewriter(cfs, txn, 1000, false, 1000000);) + SSTableRewriter rewriter = new SSTableRewriter(cfs, txn, 1000, false, 1000000); + CompactionIterator ci = new CompactionIterator(OperationType.COMPACTION, Collections.singletonList(scanner), controller, FBUtilities.nowInSeconds(), UUIDGen.getTimeUUID())) { rewriter.switchWriter(getWriter(cfs, s.descriptor.directory)); - while(scanner.hasNext()) + while(ci.hasNext()) { - rewriter.append(new LazilyCompactedRow(controller, Arrays.asList(scanner.next()))); + rewriter.append(ci.next()); if (rewriter.currentWriter().getOnDiskFilePointer() > 2500000) { assertEquals(files, cfs.getSSTables().size()); // all files are now opened early @@ -576,7 +629,6 @@ public class SSTableRewriterTest extends SchemaLoader sstables = rewriter.finish(); } - assertEquals(files, sstables.size()); assertEquals(files, cfs.getSSTables().size()); SSTableDeletingTask.waitForDeletions(); @@ -648,12 +700,13 @@ public class SSTableRewriterTest extends SchemaLoader LifecycleTransaction txn = offline ? LifecycleTransaction.offline(OperationType.UNKNOWN, compacting) : cfs.getTracker().tryModify(compacting, OperationType.UNKNOWN); SSTableRewriter rewriter = new SSTableRewriter(cfs, txn, 1000, offline, 10000000); + CompactionIterator ci = new CompactionIterator(OperationType.COMPACTION, Collections.singletonList(scanner), controller, FBUtilities.nowInSeconds(), UUIDGen.getTimeUUID()) ) { rewriter.switchWriter(getWriter(cfs, s.descriptor.directory)); - while (scanner.hasNext()) + while (ci.hasNext()) { - rewriter.append(new LazilyCompactedRow(controller, Arrays.asList(scanner.next()))); + rewriter.append(ci.next()); if (rewriter.currentWriter().getOnDiskFilePointer() > 25000000) { rewriter.switchWriter(getWriter(cfs, s.descriptor.directory)); @@ -709,11 +762,14 @@ public class SSTableRewriterTest extends SchemaLoader truncate(cfs); for (int i = 0; i < 100; i++) { - DecoratedKey key = Util.dk(Integer.toString(i)); - Mutation rm = new Mutation(KEYSPACE, key.getKey()); + String key = Integer.toString(i); + for (int j = 0; j < 10; j++) - rm.add(CF, Util.cellname(Integer.toString(j)), ByteBufferUtil.EMPTY_BYTE_BUFFER, 100); - rm.apply(); + new RowUpdateBuilder(cfs.metadata, 100, key) + .clustering(Integer.toString(j)) + .add("val", ByteBufferUtil.EMPTY_BYTE_BUFFER) + .build() + .apply(); } cfs.forceBlockingFlush(); cfs.forceMajorCompaction(); @@ -729,12 +785,13 @@ public class SSTableRewriterTest extends SchemaLoader CompactionController controller = new CompactionController(cfs, compacting, 0); LifecycleTransaction txn = cfs.getTracker().tryModify(compacting, OperationType.UNKNOWN); SSTableRewriter rewriter = new SSTableRewriter(cfs, txn, 1000, false, 1); + CompactionIterator ci = new CompactionIterator(OperationType.COMPACTION, Collections.singletonList(scanner), controller, FBUtilities.nowInSeconds(), UUIDGen.getTimeUUID()) ) { rewriter.switchWriter(getWriter(cfs, s.descriptor.directory)); - while (scanner.hasNext()) + while (ci.hasNext()) { - rewriter.append(new LazilyCompactedRow(controller, Arrays.asList(scanner.next()))); + rewriter.append(ci.next()); if (keyCount % 10 == 0) { rewriter.switchWriter(getWriter(cfs, s.descriptor.directory)); @@ -765,13 +822,13 @@ public class SSTableRewriterTest extends SchemaLoader CompactionController controller = new CompactionController(cfs, sstables, 0); LifecycleTransaction txn = cfs.getTracker().tryModify(sstables, OperationType.UNKNOWN); SSTableRewriter writer = new SSTableRewriter(cfs, txn, 1000, false, 10000000); + CompactionIterator ci = new CompactionIterator(OperationType.COMPACTION, Collections.singletonList(scanner), controller, FBUtilities.nowInSeconds(), UUIDGen.getTimeUUID()) ) { writer.switchWriter(getWriter(cfs, sstables.iterator().next().descriptor.directory)); - while (scanner.hasNext()) + while (ci.hasNext()) { - AbstractCompactedRow row = new LazilyCompactedRow(controller, Collections.singletonList(scanner.next())); - writer.append(row); + writer.append(ci.next()); if (!checked && writer.currentWriter().getFilePointer() > 15000000) { checked = true; @@ -804,30 +861,26 @@ public class SSTableRewriterTest extends SchemaLoader cfs.addSSTable(s); Set sstables = Sets.newHashSet(s); assertEquals(1, sstables.size()); + int nowInSec = FBUtilities.nowInSeconds(); try (AbstractCompactionStrategy.ScannerList scanners = cfs.getCompactionStrategyManager().getScanners(sstables); LifecycleTransaction txn = cfs.getTracker().tryModify(sstables, OperationType.UNKNOWN); SSTableRewriter writer = new SSTableRewriter(cfs, txn, 1000, false, false); - SSTableRewriter writer2 = new SSTableRewriter(cfs, txn, 1000, false, false)) + SSTableRewriter writer2 = new SSTableRewriter(cfs, txn, 1000, false, false); + CompactionController controller = new CompactionController(cfs, sstables, cfs.gcBefore(nowInSec)); + CompactionIterator ci = new CompactionIterator(OperationType.COMPACTION, scanners.scanners, controller, nowInSec, UUIDGen.getTimeUUID()) + ) { - ISSTableScanner scanner = scanners.scanners.get(0); - CompactionController controller = new CompactionController(cfs, sstables, cfs.gcBefore(System.currentTimeMillis())); writer.switchWriter(getWriter(cfs, sstables.iterator().next().descriptor.directory)); writer2.switchWriter(getWriter(cfs, sstables.iterator().next().descriptor.directory)); - while (scanner.hasNext()) + while (ci.hasNext()) { - AbstractCompactedRow row = new LazilyCompactedRow(controller, Collections.singletonList(scanner.next())); - if (writer.currentWriter().getFilePointer() < 15000000) - writer.append(row); + writer.append(ci.next()); else - writer2.append(row); + writer2.append(ci.next()); } for (int i = 0; i < 5000; i++) - { - DecoratedKey key = Util.dk(ByteBufferUtil.bytes(i)); - ColumnFamily cf = Util.getColumnFamily(keyspace, key, CF); - assertTrue(cf != null); - } + assertFalse(Util.getOnlyPartition(Util.cmd(cfs, ByteBufferUtil.bytes(i)).build()).isEmpty()); } truncateCF(); validateCFS(cfs); @@ -839,8 +892,8 @@ public class SSTableRewriterTest extends SchemaLoader for (int i = 0; i < 100; i++) { DecoratedKey key = Util.dk(Integer.toString(i)); - ColumnFamily cf = Util.getColumnFamily(ks, key, CF); - assertTrue(cf != null); + ArrayBackedPartition partition = Util.getOnlyPartitionUnfiltered(Util.cmd(ks.getColumnFamilyStore(CF), key).build()); + assertTrue(partition != null && partition.rowCount() > 0); } } @@ -866,16 +919,19 @@ public class SSTableRewriterTest extends SchemaLoader File dir = cfs.directories.getDirectoryForNewSSTables(); String filename = cfs.getTempSSTablePath(dir); - SSTableWriter writer = SSTableWriter.create(filename, 0, 0); - int end = f == fileCount - 1 ? partitionCount : ((f + 1) * partitionCount) / fileCount; - for ( ; i < end ; i++) + try (SSTableWriter writer = SSTableWriter.create(filename, 0, 0, new SerializationHeader(cfs.metadata, cfs.metadata.partitionColumns(), RowStats.NO_STATS))) { - ArrayBackedSortedColumns cf = ArrayBackedSortedColumns.factory.create(cfs.metadata); - for (int j = 0; j < cellCount ; j++) - cf.addColumn(Util.cellname(j), random(0, cellSize), 1); - writer.append(StorageService.getPartitioner().decorateKey(ByteBufferUtil.bytes(i)), cf); + int end = f == fileCount - 1 ? partitionCount : ((f + 1) * partitionCount) / fileCount; + for ( ; i < end ; i++) + { + UpdateBuilder builder = UpdateBuilder.create(cfs.metadata, ByteBufferUtil.bytes(i)); + for (int j = 0; j < cellCount ; j++) + builder.newRow(Integer.toString(i)).add("val", random(0, 1000)); + + writer.append(builder.build().unfilteredIterator()); + } + result.add(writer.finish(true)); } - result.add(writer.finish(true)); } return result; } @@ -932,7 +988,7 @@ public class SSTableRewriterTest extends SchemaLoader public static SSTableWriter getWriter(ColumnFamilyStore cfs, File directory) { String filename = cfs.getTempSSTablePath(directory); - return SSTableWriter.create(filename, 0, 0); + return SSTableWriter.create(filename, 0, 0, new SerializationHeader(cfs.metadata, cfs.metadata.partitionColumns(), RowStats.NO_STATS)); } public static ByteBuffer random(int i, int size) @@ -943,5 +999,4 @@ public class SSTableRewriterTest extends SchemaLoader r.putInt(0, i); return r; } - } diff --git a/test/unit/org/apache/cassandra/io/sstable/SSTableScannerTest.java b/test/unit/org/apache/cassandra/io/sstable/SSTableScannerTest.java index f8b808dd48..45bca51246 100644 --- a/test/unit/org/apache/cassandra/io/sstable/SSTableScannerTest.java +++ b/test/unit/org/apache/cassandra/io/sstable/SSTableScannerTest.java @@ -1,42 +1,57 @@ /* -* Licensed to the Apache Software Foundation (ASF) under one -* or more contributor license agreements. See the NOTICE file -* distributed with this work for additional information -* regarding copyright ownership. The ASF licenses this file -* to you under the Apache License, Version 2.0 (the -* "License"); you may not use this file except in compliance -* with the License. You may obtain a copy of the License at -* -* http://www.apache.org/licenses/LICENSE-2.0 -* -* Unless required by applicable law or agreed to in writing, -* software distributed under the License is distributed on an -* "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -* KIND, either express or implied. See the License for the -* specific language governing permissions and limitations -* under the License. -*/ + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ package org.apache.cassandra.io.sstable; import java.io.IOException; -import java.util.*; +import java.util.ArrayList; +import java.util.Arrays; +import java.util.Collection; +import java.util.List; -import org.apache.cassandra.io.sstable.format.SSTableReader; -import org.junit.BeforeClass; import com.google.common.collect.Iterables; +import org.junit.BeforeClass; import org.junit.Test; import org.apache.cassandra.SchemaLoader; import org.apache.cassandra.Util; +import org.apache.cassandra.config.CFMetaData; import org.apache.cassandra.config.KSMetaData; -import org.apache.cassandra.db.*; -import org.apache.cassandra.db.columniterator.IdentityQueryFilter; -import org.apache.cassandra.dht.*; +import org.apache.cassandra.db.ColumnFamilyStore; +import org.apache.cassandra.db.DataRange; +import org.apache.cassandra.db.DecoratedKey; +import org.apache.cassandra.db.Keyspace; +import org.apache.cassandra.db.PartitionPosition; +import org.apache.cassandra.db.RowUpdateBuilder; +import org.apache.cassandra.db.Slices; +import org.apache.cassandra.db.filter.ClusteringIndexSliceFilter; +import org.apache.cassandra.db.filter.ColumnFilter; +import org.apache.cassandra.dht.AbstractBounds; +import org.apache.cassandra.dht.ByteOrderedPartitioner; +import org.apache.cassandra.dht.Range; +import org.apache.cassandra.dht.Token; +import org.apache.cassandra.io.sstable.format.SSTableReader; import org.apache.cassandra.locator.SimpleStrategy; import org.apache.cassandra.utils.ByteBufferUtil; import static org.apache.cassandra.dht.AbstractBounds.isEmpty; -import static org.junit.Assert.*; +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertFalse; +import static org.junit.Assert.assertTrue; public class SSTableScannerTest { @@ -59,51 +74,51 @@ public class SSTableScannerTest } // we produce all DataRange variations that produce an inclusive start and exclusive end range - private static Iterable dataRanges(int start, int end) + private static Iterable dataRanges(CFMetaData metadata, int start, int end) { if (end < start) - return dataRanges(start, end, false, true); - return Iterables.concat(dataRanges(start, end, false, false), - dataRanges(start, end, false, true), - dataRanges(start, end, true, false), - dataRanges(start, end, true, true) + return dataRanges(metadata, start, end, false, true); + return Iterables.concat(dataRanges(metadata, start, end, false, false), + dataRanges(metadata, start, end, false, true), + dataRanges(metadata, start, end, true, false), + dataRanges(metadata, start, end, true, true) ); } - private static Iterable dataRanges(int start, int end, boolean inclusiveStart, boolean inclusiveEnd) + private static Iterable dataRanges(CFMetaData metadata, int start, int end, boolean inclusiveStart, boolean inclusiveEnd) { List ranges = new ArrayList<>(); if (start == end + 1) { assert !inclusiveStart && inclusiveEnd; - ranges.add(dataRange(min(start), false, max(end), true)); - ranges.add(dataRange(min(start), false, min(end + 1), true)); - ranges.add(dataRange(max(start - 1), false, max(end), true)); - ranges.add(dataRange(dk(start - 1), false, dk(start - 1), true)); + ranges.add(dataRange(metadata, min(start), false, max(end), true)); + ranges.add(dataRange(metadata, min(start), false, min(end + 1), true)); + ranges.add(dataRange(metadata, max(start - 1), false, max(end), true)); + ranges.add(dataRange(metadata, dk(start - 1), false, dk(start - 1), true)); } else { - for (RowPosition s : starts(start, inclusiveStart)) + for (PartitionPosition s : starts(start, inclusiveStart)) { - for (RowPosition e : ends(end, inclusiveEnd)) + for (PartitionPosition e : ends(end, inclusiveEnd)) { if (end < start && e.compareTo(s) > 0) continue; if (!isEmpty(new AbstractBounds.Boundary<>(s, inclusiveStart), new AbstractBounds.Boundary<>(e, inclusiveEnd))) continue; - ranges.add(dataRange(s, inclusiveStart, e, inclusiveEnd)); + ranges.add(dataRange(metadata, s, inclusiveStart, e, inclusiveEnd)); } } } return ranges; } - private static Iterable starts(int key, boolean inclusive) + private static Iterable starts(int key, boolean inclusive) { return Arrays.asList(min(key), max(key - 1), dk(inclusive ? key : key - 1)); } - private static Iterable ends(int key, boolean inclusive) + private static Iterable ends(int key, boolean inclusive) { return Arrays.asList(max(key), min(key + 1), dk(inclusive ? key : key + 1)); } @@ -118,19 +133,22 @@ public class SSTableScannerTest return key == Integer.MIN_VALUE ? ByteOrderedPartitioner.MINIMUM : new ByteOrderedPartitioner.BytesToken(toKey(key).getBytes()); } - private static RowPosition min(int key) + private static PartitionPosition min(int key) { return token(key).minKeyBound(); } - private static RowPosition max(int key) + private static PartitionPosition max(int key) { return token(key).maxKeyBound(); } - private static DataRange dataRange(RowPosition start, boolean startInclusive, RowPosition end, boolean endInclusive) + private static DataRange dataRange(CFMetaData metadata, PartitionPosition start, boolean startInclusive, PartitionPosition end, boolean endInclusive) { - return new DataRange(AbstractBounds.bounds(start, startInclusive, end, endInclusive), new IdentityQueryFilter()); + Slices.Builder sb = new Slices.Builder(metadata.comparator); + ClusteringIndexSliceFilter filter = new ClusteringIndexSliceFilter(sb.build(), false); + + return new DataRange(AbstractBounds.bounds(start, startInclusive, end, endInclusive), filter); } private static Range rangeFor(int start, int end) @@ -147,25 +165,28 @@ public class SSTableScannerTest return ranges; } - private static void insertRowWithKey(int key) + private static void insertRowWithKey(CFMetaData metadata, int key) { long timestamp = System.currentTimeMillis(); - DecoratedKey decoratedKey = Util.dk(toKey(key)); - Mutation rm = new Mutation(KEYSPACE, decoratedKey.getKey()); - rm.add(TABLE, Util.cellname("col"), ByteBufferUtil.EMPTY_BYTE_BUFFER, timestamp, 1000); - rm.applyUnsafe(); + + new RowUpdateBuilder(metadata, timestamp, toKey(key)) + .clustering("col") + .add("val", ByteBufferUtil.EMPTY_BYTE_BUFFER) + .build() + .applyUnsafe(); + } private static void assertScanMatches(SSTableReader sstable, int scanStart, int scanEnd, int ... boundaries) { assert boundaries.length % 2 == 0; - for (DataRange range : dataRanges(scanStart, scanEnd)) + for (DataRange range : dataRanges(sstable.metadata, scanStart, scanEnd)) { - try(ISSTableScanner scanner = sstable.getScanner(range)) + try(ISSTableScanner scanner = sstable.getScanner(ColumnFilter.all(sstable.metadata), range, false)) { for (int b = 0; b < boundaries.length; b += 2) for (int i = boundaries[b]; i <= boundaries[b + 1]; i++) - assertEquals(toKey(i), new String(scanner.next().getKey().getKey().array())); + assertEquals(toKey(i), new String(scanner.next().partitionKey().getKey().array())); assertFalse(scanner.hasNext()); } catch (Exception e) @@ -191,16 +212,16 @@ public class SSTableScannerTest store.disableAutoCompaction(); for (int i = 2; i < 10; i++) - insertRowWithKey(i); + insertRowWithKey(store.metadata, i); store.forceBlockingFlush(); assertEquals(1, store.getSSTables().size()); SSTableReader sstable = store.getSSTables().iterator().next(); // full range scan - ISSTableScanner scanner = sstable.getScanner(); + ISSTableScanner scanner = sstable.getScanner(null); for (int i = 2; i < 10; i++) - assertEquals(toKey(i), new String(scanner.next().getKey().getKey().array())); + assertEquals(toKey(i), new String(scanner.next().partitionKey().getKey().array())); scanner.close(); @@ -278,7 +299,7 @@ public class SSTableScannerTest for (int expected = rangeStart; expected <= rangeEnd; expected++) { assertTrue(String.format("Expected to see key %03d", expected), scanner.hasNext()); - assertEquals(toKey(expected), new String(scanner.next().getKey().getKey().array())); + assertEquals(toKey(expected), new String(scanner.next().partitionKey().getKey().array())); } } assertFalse(scanner.hasNext()); @@ -297,14 +318,14 @@ public class SSTableScannerTest for (int i = 0; i < 3; i++) for (int j = 2; j < 10; j++) - insertRowWithKey(i * 100 + j); + insertRowWithKey(store.metadata, i * 100 + j); store.forceBlockingFlush(); assertEquals(1, store.getSSTables().size()); SSTableReader sstable = store.getSSTables().iterator().next(); // full range scan - ISSTableScanner fullScanner = sstable.getScanner(); + ISSTableScanner fullScanner = sstable.getScanner(null); assertScanContainsRanges(fullScanner, 2, 9, 102, 109, @@ -427,14 +448,14 @@ public class SSTableScannerTest // disable compaction while flushing store.disableAutoCompaction(); - insertRowWithKey(205); + insertRowWithKey(store.metadata, 205); store.forceBlockingFlush(); assertEquals(1, store.getSSTables().size()); SSTableReader sstable = store.getSSTables().iterator().next(); // full range scan - ISSTableScanner fullScanner = sstable.getScanner(); + ISSTableScanner fullScanner = sstable.getScanner(null); assertScanContainsRanges(fullScanner, 205, 205); // scan three ranges separately diff --git a/test/unit/org/apache/cassandra/io/sstable/SSTableSimpleWriterTest.java b/test/unit/org/apache/cassandra/io/sstable/SSTableSimpleWriterTest.java deleted file mode 100644 index 499caf774f..0000000000 --- a/test/unit/org/apache/cassandra/io/sstable/SSTableSimpleWriterTest.java +++ /dev/null @@ -1,123 +0,0 @@ -/** - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - */ -package org.apache.cassandra.io.sstable; - -import java.io.File; - -import org.junit.BeforeClass; -import org.junit.Test; - -import org.apache.cassandra.SchemaLoader; -import org.apache.cassandra.Util; -import org.apache.cassandra.config.KSMetaData; -import org.apache.cassandra.config.Schema; -import org.apache.cassandra.db.*; -import org.apache.cassandra.db.marshal.IntegerType; -import org.apache.cassandra.dht.IPartitioner; -import org.apache.cassandra.locator.SimpleStrategy; -import org.apache.cassandra.service.StorageService; -import static org.apache.cassandra.utils.ByteBufferUtil.bytes; -import static org.apache.cassandra.utils.ByteBufferUtil.toInt; - -public class SSTableSimpleWriterTest -{ - public static final String KEYSPACE = "SSTableSimpleWriterTest"; - public static final String CF_STANDARDINT = "StandardInteger1"; - - @BeforeClass - public static void defineSchema() throws Exception - { - SchemaLoader.prepareServer(); - SchemaLoader.createKeyspace(KEYSPACE, - SimpleStrategy.class, - KSMetaData.optsWithRF(1), - SchemaLoader.standardCFMD(KEYSPACE, CF_STANDARDINT)); - } - - @Test - public void testSSTableSimpleUnsortedWriter() throws Exception - { - final int INC = 5; - final int NBCOL = 10; - - String keyspaceName = KEYSPACE; - String cfname = "StandardInteger1"; - - Keyspace t = Keyspace.open(keyspaceName); // make sure we create the directory - File dir = new Directories(Schema.instance.getCFMetaData(keyspaceName, cfname)).getDirectoryForNewSSTables(); - assert dir.exists(); - - IPartitioner partitioner = StorageService.getPartitioner(); - try (SSTableSimpleUnsortedWriter writer = new SSTableSimpleUnsortedWriter(dir, partitioner, keyspaceName, cfname, IntegerType.instance, null, 16)) - { - - int k = 0; - - // Adding a few rows first - for (; k < 10; ++k) - { - writer.newRow(bytes("Key" + k)); - writer.addColumn(bytes(1), bytes("v"), 0); - writer.addColumn(bytes(2), bytes("v"), 0); - writer.addColumn(bytes(3), bytes("v"), 0); - } - - - // Testing multiple opening of the same row - // We'll write column 0, 5, 10, .., on the first row, then 1, 6, 11, ... on the second one, etc. - for (int i = 0; i < INC; ++i) - { - writer.newRow(bytes("Key" + k)); - for (int j = 0; j < NBCOL; ++j) - { - writer.addColumn(bytes(i + INC * j), bytes("v"), 1); - } - } - k++; - - // Adding a few more rows - for (; k < 20; ++k) - { - writer.newRow(bytes("Key" + k)); - writer.addColumn(bytes(1), bytes("v"), 0); - writer.addColumn(bytes(2), bytes("v"), 0); - writer.addColumn(bytes(3), bytes("v"), 0); - } - } - - // Now add that newly created files to the column family - ColumnFamilyStore cfs = t.getColumnFamilyStore(cfname); - cfs.loadNewSSTables(); - - // Check we get expected results - ColumnFamily cf = Util.getColumnFamily(t, Util.dk("Key10"), cfname); - assert cf.getColumnCount() == INC * NBCOL : "expecting " + (INC * NBCOL) + " columns, got " + cf.getColumnCount(); - int i = 0; - for (Cell c : cf) - { - assert toInt(c.name().toByteBuffer()) == i : "Cell name should be " + i + ", got " + toInt(c.name().toByteBuffer()); - assert c.value().equals(bytes("v")); - assert c.timestamp() == 1; - ++i; - } - - cf = Util.getColumnFamily(t, Util.dk("Key19"), cfname); - assert cf.getColumnCount() == 3 : "expecting 3 columns, got " + cf.getColumnCount(); - } -} diff --git a/test/unit/org/apache/cassandra/io/sstable/SSTableUtils.java b/test/unit/org/apache/cassandra/io/sstable/SSTableUtils.java index a116b8409b..2c8377f27b 100644 --- a/test/unit/org/apache/cassandra/io/sstable/SSTableUtils.java +++ b/test/unit/org/apache/cassandra/io/sstable/SSTableUtils.java @@ -23,12 +23,15 @@ import java.io.File; import java.io.IOException; import java.util.*; +import org.apache.cassandra.config.Schema; import org.apache.cassandra.db.*; -import org.apache.cassandra.db.columniterator.OnDiskAtomIterator; +import org.apache.cassandra.db.rows.*; +import org.apache.cassandra.db.partitions.*; import org.apache.cassandra.io.sstable.format.SSTableReader; import org.apache.cassandra.io.sstable.format.SSTableWriter; import org.apache.cassandra.service.ActiveRepairService; import org.apache.cassandra.utils.ByteBufferUtil; +import org.apache.cassandra.utils.FBUtilities; import org.apache.cassandra.Util; import static org.junit.Assert.assertEquals; @@ -45,7 +48,7 @@ public class SSTableUtils CFNAME = cfname; } - /**/ + /* public static ColumnFamily createCF(long mfda, int ldt, Cell... cols) { return createCF(KEYSPACENAME, CFNAME, mfda, ldt, cols); @@ -64,6 +67,7 @@ public class SSTableUtils { return tempSSTableFile(keyspaceName, cfname, 0); } + */ public static File tempSSTableFile(String keyspaceName, String cfname, int generation) throws IOException { @@ -88,40 +92,29 @@ public class SSTableUtils { while (slhs.hasNext()) { - OnDiskAtomIterator ilhs = slhs.next(); + UnfilteredRowIterator ilhs = slhs.next(); assert srhs.hasNext() : "LHS contained more rows than RHS"; - OnDiskAtomIterator irhs = srhs.next(); + UnfilteredRowIterator irhs = srhs.next(); assertContentEquals(ilhs, irhs); } assert !srhs.hasNext() : "RHS contained more rows than LHS"; } } - public static void assertContentEquals(OnDiskAtomIterator lhs, OnDiskAtomIterator rhs) + public static void assertContentEquals(UnfilteredRowIterator lhs, UnfilteredRowIterator rhs) { - assertEquals(lhs.getKey(), rhs.getKey()); - // check metadata - ColumnFamily lcf = lhs.getColumnFamily(); - ColumnFamily rcf = rhs.getColumnFamily(); - if (lcf == null) - { - if (rcf == null) - return; - throw new AssertionError("LHS had no content for " + rhs.getKey()); - } - else if (rcf == null) - throw new AssertionError("RHS had no content for " + lhs.getKey()); - assertEquals(lcf.deletionInfo(), rcf.deletionInfo()); + assertEquals(lhs.partitionKey(), rhs.partitionKey()); + assertEquals(lhs.partitionLevelDeletion(), rhs.partitionLevelDeletion()); // iterate columns while (lhs.hasNext()) { - Cell clhs = (Cell)lhs.next(); - assert rhs.hasNext() : "LHS contained more columns than RHS for " + lhs.getKey(); - Cell crhs = (Cell)rhs.next(); + Unfiltered clhs = lhs.next(); + assert rhs.hasNext() : "LHS contained more columns than RHS for " + lhs.partitionKey(); + Unfiltered crhs = rhs.next(); - assertEquals("Mismatched columns for " + lhs.getKey(), clhs, crhs); + assertEquals("Mismatched row/tombstone for " + lhs.partitionKey(), clhs, crhs); } - assert !rhs.hasNext() : "RHS contained more columns than LHS for " + lhs.getKey(); + assert !rhs.hasNext() : "RHS contained more columns than LHS for " + lhs.partitionKey(); } /** @@ -176,19 +169,19 @@ public class SSTableUtils public SSTableReader write(Set keys) throws IOException { - Map map = new HashMap(); + Map map = new HashMap<>(); for (String key : keys) { - ColumnFamily cf = ArrayBackedSortedColumns.factory.create(ksname, cfname); - cf.addColumn(new BufferCell(Util.cellname(key), ByteBufferUtil.bytes(key), 0)); - map.put(key, cf); + RowUpdateBuilder builder = new RowUpdateBuilder(Schema.instance.getCFMetaData(ksname, cfname), 0, key); + builder.clustering(key).add("val", key); + map.put(key, builder.buildUpdate()); } return write(map); } - public SSTableReader write(SortedMap sorted) throws IOException + public SSTableReader write(SortedMap sorted) throws IOException { - final Iterator> iter = sorted.entrySet().iterator(); + final Iterator> iter = sorted.entrySet().iterator(); return write(sorted.size(), new Appender() { @Override @@ -196,17 +189,16 @@ public class SSTableUtils { if (!iter.hasNext()) return false; - Map.Entry entry = iter.next(); - writer.append(entry.getKey(), entry.getValue()); + writer.append(iter.next().getValue().unfilteredIterator()); return true; } }); } - public SSTableReader write(Map entries) throws IOException + public SSTableReader write(Map entries) throws IOException { - SortedMap sorted = new TreeMap(); - for (Map.Entry entry : entries.entrySet()) + SortedMap sorted = new TreeMap<>(); + for (Map.Entry entry : entries.entrySet()) sorted.put(Util.dk(entry.getKey()), entry.getValue()); return write(sorted); @@ -215,7 +207,8 @@ public class SSTableUtils public SSTableReader write(int expectedSize, Appender appender) throws IOException { File datafile = (dest == null) ? tempSSTableFile(ksname, cfname, generation) : new File(dest.filenameFor(Component.DATA)); - SSTableWriter writer = SSTableWriter.create(Descriptor.fromFilename(datafile.getAbsolutePath()), expectedSize, ActiveRepairService.UNREPAIRED_SSTABLE, 0); + SerializationHeader header = SerializationHeader.make(Schema.instance.getCFMetaData(ksname, cfname), Collections.EMPTY_LIST); + SSTableWriter writer = SSTableWriter.create(Descriptor.fromFilename(datafile.getAbsolutePath()), expectedSize, ActiveRepairService.UNREPAIRED_SSTABLE, 0, header); while (appender.append(writer)) { /* pass */ } SSTableReader reader = writer.finish(true); // mark all components for removal diff --git a/test/unit/org/apache/cassandra/io/sstable/metadata/MetadataSerializerTest.java b/test/unit/org/apache/cassandra/io/sstable/metadata/MetadataSerializerTest.java index eda4f173e7..7051bd35f7 100644 --- a/test/unit/org/apache/cassandra/io/sstable/metadata/MetadataSerializerTest.java +++ b/test/unit/org/apache/cassandra/io/sstable/metadata/MetadataSerializerTest.java @@ -15,26 +15,29 @@ * See the License for the specific language governing permissions and * limitations under the License. */ + package org.apache.cassandra.io.sstable.metadata; import java.io.File; import java.io.FileOutputStream; import java.io.IOException; +import java.util.Collections; import java.util.EnumSet; import java.util.Map; import java.util.Set; import com.google.common.collect.Sets; - import org.junit.Test; + +import org.apache.cassandra.SchemaLoader; +import org.apache.cassandra.config.CFMetaData; +import org.apache.cassandra.db.SerializationHeader; import org.apache.cassandra.db.commitlog.ReplayPosition; -import org.apache.cassandra.db.composites.SimpleDenseCellNameType; -import org.apache.cassandra.db.marshal.BytesType; import org.apache.cassandra.dht.RandomPartitioner; import org.apache.cassandra.io.sstable.Component; import org.apache.cassandra.io.sstable.Descriptor; -import org.apache.cassandra.io.util.DataOutputStreamPlus; import org.apache.cassandra.io.util.BufferedDataOutputStreamPlus; +import org.apache.cassandra.io.util.DataOutputStreamPlus; import org.apache.cassandra.io.util.RandomAccessReader; import org.apache.cassandra.utils.EstimatedHistogram; @@ -45,20 +48,15 @@ public class MetadataSerializerTest @Test public void testSerialization() throws IOException { - EstimatedHistogram rowSizes = new EstimatedHistogram(new long[] { 1L, 2L }, - new long[] { 3L, 4L, 5L }); - EstimatedHistogram columnCounts = new EstimatedHistogram(new long[] { 6L, 7L }, - new long[] { 8L, 9L, 10L }); - ReplayPosition rp = new ReplayPosition(11L, 12); - long minTimestamp = 2162517136L; - long maxTimestamp = 4162517136L; - MetadataCollector collector = new MetadataCollector(new SimpleDenseCellNameType(BytesType.instance)) - .estimatedRowSize(rowSizes) - .estimatedColumnCount(columnCounts) - .replayPosition(rp); - collector.updateMinTimestamp(minTimestamp); - collector.updateMaxTimestamp(maxTimestamp); + CFMetaData cfm = SchemaLoader.standardCFMD("ks1", "cf1"); + + + ReplayPosition rp = new ReplayPosition(11L, 12); + + + MetadataCollector collector = new MetadataCollector(cfm.comparator).replayPosition(rp); + Set ancestors = Sets.newHashSet(1, 2, 3, 4); for (int i : ancestors) @@ -66,7 +64,7 @@ public class MetadataSerializerTest String partitioner = RandomPartitioner.class.getCanonicalName(); double bfFpChance = 0.1; - Map originalMetadata = collector.finalizeMetadata(partitioner, bfFpChance, 0); + Map originalMetadata = collector.finalizeMetadata(partitioner, bfFpChance, 0, SerializationHeader.make(cfm, Collections.EMPTY_LIST)); MetadataSerializer serializer = new MetadataSerializer(); // Serialize to tmp file diff --git a/test/unit/org/apache/cassandra/locator/CloudstackSnitchTest.java b/test/unit/org/apache/cassandra/locator/CloudstackSnitchTest.java index d9a4ef1d0c..7881265d0e 100644 --- a/test/unit/org/apache/cassandra/locator/CloudstackSnitchTest.java +++ b/test/unit/org/apache/cassandra/locator/CloudstackSnitchTest.java @@ -15,6 +15,7 @@ * See the License for the specific language governing permissions and * limitations under the License. */ + package org.apache.cassandra.locator; import java.io.IOException; @@ -26,12 +27,12 @@ import org.junit.BeforeClass; import org.junit.Test; import org.apache.cassandra.SchemaLoader; +import org.apache.cassandra.db.Keyspace; import org.apache.cassandra.exceptions.ConfigurationException; import org.apache.cassandra.gms.ApplicationState; import org.apache.cassandra.gms.Gossiper; import org.apache.cassandra.gms.VersionedValue; import org.apache.cassandra.service.StorageService; -import org.apache.cassandra.db.Keyspace; import static org.junit.Assert.assertEquals; diff --git a/test/unit/org/apache/cassandra/locator/DynamicEndpointSnitchTest.java b/test/unit/org/apache/cassandra/locator/DynamicEndpointSnitchTest.java index c7c1f174cf..a928a3032a 100644 --- a/test/unit/org/apache/cassandra/locator/DynamicEndpointSnitchTest.java +++ b/test/unit/org/apache/cassandra/locator/DynamicEndpointSnitchTest.java @@ -1,21 +1,20 @@ /* -* Licensed to the Apache Software Foundation (ASF) under one -* or more contributor license agreements. See the NOTICE file -* distributed with this work for additional information -* regarding copyright ownership. The ASF licenses this file -* to you under the Apache License, Version 2.0 (the -* "License"); you may not use this file except in compliance -* with the License. You may obtain a copy of the License at -* -* http://www.apache.org/licenses/LICENSE-2.0 -* -* Unless required by applicable law or agreed to in writing, -* software distributed under the License is distributed on an -* "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -* KIND, either express or implied. See the License for the -* specific language governing permissions and limitations -* under the License. -*/ + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ package org.apache.cassandra.locator; @@ -24,10 +23,10 @@ import java.net.InetAddress; import java.util.Arrays; import java.util.List; -import org.apache.cassandra.exceptions.ConfigurationException; -import org.apache.cassandra.service.StorageService; import org.junit.Test; +import org.apache.cassandra.exceptions.ConfigurationException; +import org.apache.cassandra.service.StorageService; import org.apache.cassandra.utils.FBUtilities; import static org.junit.Assert.assertEquals; diff --git a/test/unit/org/apache/cassandra/locator/EC2SnitchTest.java b/test/unit/org/apache/cassandra/locator/EC2SnitchTest.java index 6015adf1c9..cb30dc0d22 100644 --- a/test/unit/org/apache/cassandra/locator/EC2SnitchTest.java +++ b/test/unit/org/apache/cassandra/locator/EC2SnitchTest.java @@ -1,6 +1,4 @@ -package org.apache.cassandra.locator; /* - * * Licensed to the Apache Software Foundation (ASF) under one * or more contributor license agreements. See the NOTICE file * distributed with this work for additional information @@ -9,17 +7,17 @@ package org.apache.cassandra.locator; * "License"); you may not use this file except in compliance * with the License. You may obtain a copy of the License at * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. + * http://www.apache.org/licenses/LICENSE-2.0 * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. */ +package org.apache.cassandra.locator; + import java.io.IOException; import java.net.InetAddress; diff --git a/test/unit/org/apache/cassandra/locator/GoogleCloudSnitchTest.java b/test/unit/org/apache/cassandra/locator/GoogleCloudSnitchTest.java index 54ea722b2f..fff880d0c2 100644 --- a/test/unit/org/apache/cassandra/locator/GoogleCloudSnitchTest.java +++ b/test/unit/org/apache/cassandra/locator/GoogleCloudSnitchTest.java @@ -1,6 +1,4 @@ -package org.apache.cassandra.locator; /* - * * Licensed to the Apache Software Foundation (ASF) under one * or more contributor license agreements. See the NOTICE file * distributed with this work for additional information @@ -9,17 +7,17 @@ package org.apache.cassandra.locator; * "License"); you may not use this file except in compliance * with the License. You may obtain a copy of the License at * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. + * http://www.apache.org/licenses/LICENSE-2.0 * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. */ +package org.apache.cassandra.locator; + import java.io.IOException; import java.net.InetAddress; diff --git a/test/unit/org/apache/cassandra/locator/GossipingPropertyFileSnitchTest.java b/test/unit/org/apache/cassandra/locator/GossipingPropertyFileSnitchTest.java index 16557b36fc..899c75df2c 100644 --- a/test/unit/org/apache/cassandra/locator/GossipingPropertyFileSnitchTest.java +++ b/test/unit/org/apache/cassandra/locator/GossipingPropertyFileSnitchTest.java @@ -15,6 +15,7 @@ * See the License for the specific language governing permissions and * limitations under the License. */ + package org.apache.cassandra.locator; import java.net.InetAddress; @@ -23,10 +24,11 @@ import java.nio.file.Path; import java.nio.file.Paths; import com.google.common.net.InetAddresses; -import org.apache.cassandra.utils.FBUtilities; import org.junit.Assert; import org.junit.Test; +import org.apache.cassandra.utils.FBUtilities; + /** * Unit tests for {@link GossipingPropertyFileSnitch}. */ diff --git a/test/unit/org/apache/cassandra/locator/NetworkTopologyStrategyTest.java b/test/unit/org/apache/cassandra/locator/NetworkTopologyStrategyTest.java index a3ac416bdb..bbfdd3b208 100644 --- a/test/unit/org/apache/cassandra/locator/NetworkTopologyStrategyTest.java +++ b/test/unit/org/apache/cassandra/locator/NetworkTopologyStrategyTest.java @@ -1,21 +1,20 @@ /* -* Licensed to the Apache Software Foundation (ASF) under one -* or more contributor license agreements. See the NOTICE file -* distributed with this work for additional information -* regarding copyright ownership. The ASF licenses this file -* to you under the Apache License, Version 2.0 (the -* "License"); you may not use this file except in compliance -* with the License. You may obtain a copy of the License at -* -* http://www.apache.org/licenses/LICENSE-2.0 -* -* Unless required by applicable law or agreed to in writing, -* software distributed under the License is distributed on an -* "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -* KIND, either express or implied. See the License for the -* specific language governing permissions and limitations -* under the License. -*/ + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ package org.apache.cassandra.locator; @@ -29,19 +28,17 @@ import java.util.List; import java.util.Map; import java.util.Set; +import com.google.common.collect.HashMultimap; +import com.google.common.collect.Multimap; import org.junit.Assert; - import org.junit.Test; import org.slf4j.Logger; import org.slf4j.LoggerFactory; -import org.apache.cassandra.exceptions.ConfigurationException; import org.apache.cassandra.config.DatabaseDescriptor; import org.apache.cassandra.dht.OrderPreservingPartitioner.StringToken; import org.apache.cassandra.dht.Token; - -import com.google.common.collect.HashMultimap; -import com.google.common.collect.Multimap; +import org.apache.cassandra.exceptions.ConfigurationException; public class NetworkTopologyStrategyTest { diff --git a/test/unit/org/apache/cassandra/locator/OldNetworkTopologyStrategyTest.java b/test/unit/org/apache/cassandra/locator/OldNetworkTopologyStrategyTest.java index eceb84704d..aac0798e7f 100644 --- a/test/unit/org/apache/cassandra/locator/OldNetworkTopologyStrategyTest.java +++ b/test/unit/org/apache/cassandra/locator/OldNetworkTopologyStrategyTest.java @@ -1,26 +1,23 @@ /* -* Licensed to the Apache Software Foundation (ASF) under one -* or more contributor license agreements. See the NOTICE file -* distributed with this work for additional information -* regarding copyright ownership. The ASF licenses this file -* to you under the Apache License, Version 2.0 (the -* "License"); you may not use this file except in compliance -* with the License. You may obtain a copy of the License at -* -* http://www.apache.org/licenses/LICENSE-2.0 -* -* Unless required by applicable law or agreed to in writing, -* software distributed under the License is distributed on an -* "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -* KIND, either express or implied. See the License for the -* specific language governing permissions and limitations -* under the License. -*/ + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ package org.apache.cassandra.locator; -import static org.junit.Assert.assertEquals; - import java.net.InetAddress; import java.net.UnknownHostException; import java.util.ArrayList; @@ -31,6 +28,9 @@ import java.util.List; import java.util.Map; import java.util.Set; +import org.junit.Before; +import org.junit.Test; + import org.apache.cassandra.config.KSMetaData; import org.apache.cassandra.dht.RandomPartitioner.BigIntegerToken; import org.apache.cassandra.dht.Range; @@ -38,8 +38,7 @@ import org.apache.cassandra.dht.Token; import org.apache.cassandra.service.StorageService; import org.apache.cassandra.utils.Pair; -import org.junit.Before; -import org.junit.Test; +import static org.junit.Assert.assertEquals; public class OldNetworkTopologyStrategyTest { @@ -58,7 +57,7 @@ public class OldNetworkTopologyStrategyTest /** * 4 same rack endpoints * - * @throws UnknownHostException + * @throws java.net.UnknownHostException */ @Test public void testBigIntegerEndpointsA() throws UnknownHostException @@ -83,7 +82,7 @@ public class OldNetworkTopologyStrategyTest * 3 same rack endpoints * 1 external datacenter * - * @throws UnknownHostException + * @throws java.net.UnknownHostException */ @Test public void testBigIntegerEndpointsB() throws UnknownHostException @@ -109,7 +108,7 @@ public class OldNetworkTopologyStrategyTest * 1 same datacenter, different rack endpoints * 1 external datacenter * - * @throws UnknownHostException + * @throws java.net.UnknownHostException */ @Test public void testBigIntegerEndpointsC() throws UnknownHostException @@ -167,7 +166,7 @@ public class OldNetworkTopologyStrategyTest /** * test basic methods to move a node. For sure, it's not the best place, but it's easy to test * - * @throws UnknownHostException + * @throws java.net.UnknownHostException */ @Test public void testMoveLeft() throws UnknownHostException diff --git a/test/unit/org/apache/cassandra/locator/ReplicationStrategyEndpointCacheTest.java b/test/unit/org/apache/cassandra/locator/ReplicationStrategyEndpointCacheTest.java index 093de9ba33..de2b02a2bd 100644 --- a/test/unit/org/apache/cassandra/locator/ReplicationStrategyEndpointCacheTest.java +++ b/test/unit/org/apache/cassandra/locator/ReplicationStrategyEndpointCacheTest.java @@ -1,26 +1,28 @@ /* -* Licensed to the Apache Software Foundation (ASF) under one -* or more contributor license agreements. See the NOTICE file -* distributed with this work for additional information -* regarding copyright ownership. The ASF licenses this file -* to you under the Apache License, Version 2.0 (the -* "License"); you may not use this file except in compliance -* with the License. You may obtain a copy of the License at -* -* http://www.apache.org/licenses/LICENSE-2.0 -* -* Unless required by applicable law or agreed to in writing, -* software distributed under the License is distributed on an -* "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -* KIND, either express or implied. See the License for the -* specific language governing permissions and limitations -* under the License. -*/ + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ package org.apache.cassandra.locator; import java.net.InetAddress; -import java.util.*; +import java.util.ArrayList; +import java.util.HashMap; +import java.util.List; +import java.util.Map; import org.apache.commons.lang3.StringUtils; import org.junit.BeforeClass; @@ -29,9 +31,9 @@ import org.junit.Test; import org.apache.cassandra.SchemaLoader; import org.apache.cassandra.config.KSMetaData; import org.apache.cassandra.db.Keyspace; -import org.apache.cassandra.exceptions.ConfigurationException; import org.apache.cassandra.dht.RandomPartitioner.BigIntegerToken; import org.apache.cassandra.dht.Token; +import org.apache.cassandra.exceptions.ConfigurationException; public class ReplicationStrategyEndpointCacheTest { @@ -175,11 +177,11 @@ public class ReplicationStrategyEndpointCacheTest private AbstractReplicationStrategy getStrategyWithNewTokenMetadata(AbstractReplicationStrategy strategy, TokenMetadata newTmd) throws ConfigurationException { return AbstractReplicationStrategy.createReplicationStrategy( - strategy.keyspaceName, - AbstractReplicationStrategy.getClass(strategy.getClass().getName()), - newTmd, - strategy.snitch, - strategy.configOptions); + strategy.keyspaceName, + AbstractReplicationStrategy.getClass(strategy.getClass().getName()), + newTmd, + strategy.snitch, + strategy.configOptions); } } diff --git a/test/unit/org/apache/cassandra/locator/SimpleStrategyTest.java b/test/unit/org/apache/cassandra/locator/SimpleStrategyTest.java index 61255f32f7..a806cda603 100644 --- a/test/unit/org/apache/cassandra/locator/SimpleStrategyTest.java +++ b/test/unit/org/apache/cassandra/locator/SimpleStrategyTest.java @@ -1,21 +1,20 @@ /* -* Licensed to the Apache Software Foundation (ASF) under one -* or more contributor license agreements. See the NOTICE file -* distributed with this work for additional information -* regarding copyright ownership. The ASF licenses this file -* to you under the Apache License, Version 2.0 (the -* "License"); you may not use this file except in compliance -* with the License. You may obtain a copy of the License at -* -* http://www.apache.org/licenses/LICENSE-2.0 -* -* Unless required by applicable law or agreed to in writing, -* software distributed under the License is distributed on an -* "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -* KIND, either express or implied. See the License for the -* specific language governing permissions and limitations -* under the License. -*/ + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ package org.apache.cassandra.locator; @@ -33,14 +32,18 @@ import org.apache.cassandra.SchemaLoader; import org.apache.cassandra.config.KSMetaData; import org.apache.cassandra.config.Schema; import org.apache.cassandra.db.Keyspace; -import org.apache.cassandra.dht.*; -import org.apache.cassandra.dht.RandomPartitioner.BigIntegerToken; +import org.apache.cassandra.dht.IPartitioner; +import org.apache.cassandra.dht.OrderPreservingPartitioner; import org.apache.cassandra.dht.OrderPreservingPartitioner.StringToken; +import org.apache.cassandra.dht.RandomPartitioner.BigIntegerToken; +import org.apache.cassandra.dht.Token; import org.apache.cassandra.service.PendingRangeCalculatorService; import org.apache.cassandra.service.StorageServiceAccessor; import org.apache.cassandra.utils.ByteBufferUtil; -import static org.junit.Assert.*; +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertFalse; +import static org.junit.Assert.assertTrue; public class SimpleStrategyTest { @@ -82,7 +85,7 @@ public class SimpleStrategyTest List keyTokens = new ArrayList(); for (int i = 0; i < 5; i++) { endpointTokens.add(new StringToken(String.valueOf((char)('a' + i * 2)))); - keyTokens.add(partitioner.getToken(ByteBufferUtil.bytes(String.valueOf((char)('a' + i * 2 + 1))))); + keyTokens.add(partitioner.getToken(ByteBufferUtil.bytes(String.valueOf((char) ('a' + i * 2 + 1))))); } verifyGetNaturalEndpoints(endpointTokens.toArray(new Token[0]), keyTokens.toArray(new Token[0])); } @@ -182,10 +185,10 @@ public class SimpleStrategyTest { KSMetaData ksmd = Schema.instance.getKSMetaData(keyspaceName); return AbstractReplicationStrategy.createReplicationStrategy( - keyspaceName, - ksmd.strategyClass, - tmd, - new SimpleSnitch(), - ksmd.strategyOptions); + keyspaceName, + ksmd.strategyClass, + tmd, + new SimpleSnitch(), + ksmd.strategyOptions); } } diff --git a/test/unit/org/apache/cassandra/locator/TokenMetadataTest.java b/test/unit/org/apache/cassandra/locator/TokenMetadataTest.java index 95118dc3f1..68d0dfa4b8 100644 --- a/test/unit/org/apache/cassandra/locator/TokenMetadataTest.java +++ b/test/unit/org/apache/cassandra/locator/TokenMetadataTest.java @@ -1,39 +1,37 @@ /* -* Licensed to the Apache Software Foundation (ASF) under one -* or more contributor license agreements. See the NOTICE file -* distributed with this work for additional information -* regarding copyright ownership. The ASF licenses this file -* to you under the Apache License, Version 2.0 (the -* "License"); you may not use this file except in compliance -* with the License. You may obtain a copy of the License at -* -* http://www.apache.org/licenses/LICENSE-2.0 -* -* Unless required by applicable law or agreed to in writing, -* software distributed under the License is distributed on an -* "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -* KIND, either express or implied. See the License for the -* specific language governing permissions and limitations -* under the License. -*/ + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ package org.apache.cassandra.locator; import java.net.InetAddress; import java.util.ArrayList; -import com.google.common.collect.Iterators; +import com.google.common.collect.Iterators; import org.junit.BeforeClass; import org.junit.Test; import org.junit.runner.RunWith; -import static org.junit.Assert.assertEquals; - -import static org.apache.cassandra.Util.token; import org.apache.cassandra.OrderedJUnit4ClassRunner; import org.apache.cassandra.dht.Token; -import org.apache.cassandra.locator.TokenMetadata; import org.apache.cassandra.service.StorageService; +import static org.apache.cassandra.Util.token; +import static org.junit.Assert.assertEquals; + @RunWith(OrderedJUnit4ClassRunner.class) public class TokenMetadataTest { diff --git a/test/unit/org/apache/cassandra/metrics/CQLMetricsTest.java b/test/unit/org/apache/cassandra/metrics/CQLMetricsTest.java index a357d24d73..2864b6ed95 100644 --- a/test/unit/org/apache/cassandra/metrics/CQLMetricsTest.java +++ b/test/unit/org/apache/cassandra/metrics/CQLMetricsTest.java @@ -15,14 +15,18 @@ * See the License for the specific language governing permissions and * limitations under the License. */ + package org.apache.cassandra.metrics; import java.io.IOException; +import org.junit.BeforeClass; +import org.junit.Test; +import org.junit.runner.RunWith; + import com.datastax.driver.core.Cluster; import com.datastax.driver.core.PreparedStatement; import com.datastax.driver.core.Session; - import org.apache.cassandra.OrderedJUnit4ClassRunner; import org.apache.cassandra.SchemaLoader; import org.apache.cassandra.config.DatabaseDescriptor; @@ -33,10 +37,6 @@ import org.apache.cassandra.service.EmbeddedCassandraService; import static junit.framework.Assert.assertEquals; -import org.junit.BeforeClass; -import org.junit.Test; -import org.junit.runner.RunWith; - @RunWith(OrderedJUnit4ClassRunner.class) public class CQLMetricsTest extends SchemaLoader { diff --git a/test/unit/org/apache/cassandra/metrics/LatencyMetricsTest.java b/test/unit/org/apache/cassandra/metrics/LatencyMetricsTest.java index ae4b733842..62cb88e43c 100644 --- a/test/unit/org/apache/cassandra/metrics/LatencyMetricsTest.java +++ b/test/unit/org/apache/cassandra/metrics/LatencyMetricsTest.java @@ -15,9 +15,11 @@ * See the License for the specific language governing permissions and * limitations under the License. */ + package org.apache.cassandra.metrics; import org.junit.Test; + import static junit.framework.Assert.assertFalse; public class LatencyMetricsTest diff --git a/test/unit/org/apache/cassandra/repair/LocalSyncTaskTest.java b/test/unit/org/apache/cassandra/repair/LocalSyncTaskTest.java index 3a16262509..b4d30517ab 100644 --- a/test/unit/org/apache/cassandra/repair/LocalSyncTaskTest.java +++ b/test/unit/org/apache/cassandra/repair/LocalSyncTaskTest.java @@ -15,6 +15,7 @@ * See the License for the specific language governing permissions and * limitations under the License. */ + package org.apache.cassandra.repair; import java.net.InetAddress; diff --git a/test/unit/org/apache/cassandra/repair/RepairSessionTest.java b/test/unit/org/apache/cassandra/repair/RepairSessionTest.java index 8ea2bfafdd..0af94b2d3d 100644 --- a/test/unit/org/apache/cassandra/repair/RepairSessionTest.java +++ b/test/unit/org/apache/cassandra/repair/RepairSessionTest.java @@ -15,6 +15,7 @@ * See the License for the specific language governing permissions and * limitations under the License. */ + package org.apache.cassandra.repair; import java.io.IOException; diff --git a/test/unit/org/apache/cassandra/repair/ValidatorTest.java b/test/unit/org/apache/cassandra/repair/ValidatorTest.java index a9f18f5376..f9caf4306b 100644 --- a/test/unit/org/apache/cassandra/repair/ValidatorTest.java +++ b/test/unit/org/apache/cassandra/repair/ValidatorTest.java @@ -15,14 +15,12 @@ * See the License for the specific language governing permissions and * limitations under the License. */ + package org.apache.cassandra.repair; -import java.io.IOException; import java.net.InetAddress; -import java.security.MessageDigest; import java.util.UUID; -import org.apache.cassandra.io.util.SequentialWriter; import org.junit.After; import org.junit.BeforeClass; import org.junit.Test; @@ -31,27 +29,29 @@ import org.apache.cassandra.SchemaLoader; import org.apache.cassandra.config.KSMetaData; import org.apache.cassandra.db.BufferDecoratedKey; import org.apache.cassandra.db.ColumnFamilyStore; -import org.apache.cassandra.db.DecoratedKey; import org.apache.cassandra.db.Keyspace; -import org.apache.cassandra.db.RowIndexEntry; -import org.apache.cassandra.db.compaction.AbstractCompactedRow; +import org.apache.cassandra.db.rows.UnfilteredRowIterators; import org.apache.cassandra.dht.IPartitioner; import org.apache.cassandra.dht.Range; import org.apache.cassandra.dht.Token; -import org.apache.cassandra.io.sstable.ColumnStats; import org.apache.cassandra.locator.SimpleStrategy; +import org.apache.cassandra.net.IMessageSink; import org.apache.cassandra.net.MessageIn; import org.apache.cassandra.net.MessageOut; import org.apache.cassandra.net.MessagingService; -import org.apache.cassandra.net.IMessageSink; import org.apache.cassandra.repair.messages.RepairMessage; import org.apache.cassandra.repair.messages.ValidationComplete; import org.apache.cassandra.service.StorageService; import org.apache.cassandra.utils.ByteBufferUtil; +import org.apache.cassandra.utils.FBUtilities; import org.apache.cassandra.utils.MerkleTree; import org.apache.cassandra.utils.concurrent.SimpleCondition; -import static org.junit.Assert.*; +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertFalse; +import static org.junit.Assert.assertNotNull; +import static org.junit.Assert.assertNull; +import static org.junit.Assert.assertTrue; public class ValidatorTest { @@ -124,7 +124,7 @@ public class ValidatorTest // add a row Token mid = partitioner.midpoint(range.left, range.right); - validator.add(new CompactedRowStub(new BufferDecoratedKey(mid, ByteBufferUtil.bytes("inconceivable!")))); + validator.add(UnfilteredRowIterators.emptyIterator(cfs.metadata, new BufferDecoratedKey(mid, ByteBufferUtil.bytes("inconceivable!")), false)); validator.complete(); // confirm that the tree was validated @@ -135,27 +135,6 @@ public class ValidatorTest lock.await(); } - private static class CompactedRowStub extends AbstractCompactedRow - { - private CompactedRowStub(DecoratedKey key) - { - super(key); - } - - public RowIndexEntry write(long currentPosition, SequentialWriter out) throws IOException - { - throw new UnsupportedOperationException(); - } - - public void update(MessageDigest digest) { } - - public ColumnStats columnStats() - { - throw new UnsupportedOperationException(); - } - - public void close() throws IOException { } - } @Test public void testValidatorFailed() throws Throwable diff --git a/test/unit/org/apache/cassandra/repair/messages/RepairOptionTest.java b/test/unit/org/apache/cassandra/repair/messages/RepairOptionTest.java index 557d0b12d6..47552bea90 100644 --- a/test/unit/org/apache/cassandra/repair/messages/RepairOptionTest.java +++ b/test/unit/org/apache/cassandra/repair/messages/RepairOptionTest.java @@ -15,6 +15,7 @@ * See the License for the specific language governing permissions and * limitations under the License. */ + package org.apache.cassandra.repair.messages; import java.util.HashMap; @@ -30,7 +31,9 @@ import org.apache.cassandra.dht.Range; import org.apache.cassandra.dht.Token; import org.apache.cassandra.repair.RepairParallelism; -import static org.junit.Assert.*; +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertFalse; +import static org.junit.Assert.assertTrue; public class RepairOptionTest { diff --git a/test/unit/org/apache/cassandra/schema/DefsTest.java b/test/unit/org/apache/cassandra/schema/DefsTest.java index 1e285753e2..e7eb0fe0a1 100644 --- a/test/unit/org/apache/cassandra/schema/DefsTest.java +++ b/test/unit/org/apache/cassandra/schema/DefsTest.java @@ -1,4 +1,4 @@ -/** +/* * Licensed to the Apache Software Foundation (ASF) under one * or more contributor license agreements. See the NOTICE file * distributed with this work for additional information @@ -22,44 +22,55 @@ import java.io.File; import java.nio.ByteBuffer; import java.util.function.Supplier; +import org.junit.BeforeClass; +import org.junit.Test; +import org.junit.runner.RunWith; + import org.apache.cassandra.OrderedJUnit4ClassRunner; import org.apache.cassandra.SchemaLoader; import org.apache.cassandra.Util; -import org.apache.cassandra.config.*; -import org.apache.cassandra.db.*; -import org.apache.cassandra.db.composites.*; +import org.apache.cassandra.config.CFMetaData; +import org.apache.cassandra.config.ColumnDefinition; +import org.apache.cassandra.config.IndexType; +import org.apache.cassandra.config.KSMetaData; +import org.apache.cassandra.config.Schema; +import org.apache.cassandra.cql3.QueryProcessor; +import org.apache.cassandra.cql3.UntypedResultSet; +import org.apache.cassandra.db.ColumnFamilyStore; +import org.apache.cassandra.db.DecoratedKey; +import org.apache.cassandra.db.Keyspace; import org.apache.cassandra.db.marshal.BytesType; -import org.apache.cassandra.db.marshal.UTF8Type; import org.apache.cassandra.db.marshal.TimeUUIDType; +import org.apache.cassandra.db.marshal.UTF8Type; import org.apache.cassandra.exceptions.ConfigurationException; import org.apache.cassandra.io.sstable.Component; import org.apache.cassandra.io.sstable.Descriptor; import org.apache.cassandra.io.sstable.SSTableDeletingTask; import org.apache.cassandra.locator.OldNetworkTopologyStrategy; import org.apache.cassandra.locator.SimpleStrategy; -import org.apache.cassandra.schema.LegacySchemaTables; import org.apache.cassandra.service.MigrationManager; import org.apache.cassandra.utils.ByteBufferUtil; import org.apache.cassandra.utils.FBUtilities; -import static org.apache.cassandra.Util.cellname; +import static org.apache.cassandra.cql3.CQLTester.assertRows; +import static org.apache.cassandra.cql3.CQLTester.row; +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertFalse; +import static org.junit.Assert.assertNotNull; +import static org.junit.Assert.assertNull; +import static org.junit.Assert.assertTrue; -import org.junit.Assert; -import org.junit.BeforeClass; -import org.junit.Ignore; -import org.junit.Test; -import org.junit.runner.RunWith; @RunWith(OrderedJUnit4ClassRunner.class) public class DefsTest { - private static final String KEYSPACE1 = "Keyspace1"; - private static final String KEYSPACE3 = "Keyspace3"; - private static final String KEYSPACE6 = "Keyspace6"; - private static final String EMPTYKEYSPACE = "DefsTestEmptyKeyspace"; - private static final String CF_STANDARD1 = "Standard1"; - private static final String CF_STANDARD2 = "Standard2"; - private static final String CF_INDEXED = "Indexed1"; + private static final String KEYSPACE1 = "keyspace1"; + private static final String KEYSPACE3 = "keyspace3"; + private static final String KEYSPACE6 = "keyspace6"; + private static final String EMPTY_KEYSPACE = "test_empty_keyspace"; + private static final String TABLE1 = "standard1"; + private static final String TABLE2 = "standard2"; + private static final String TABLE1i = "indexed1"; @BeforeClass public static void defineSchema() throws ConfigurationException @@ -69,26 +80,26 @@ public class DefsTest SchemaLoader.createKeyspace(KEYSPACE1, SimpleStrategy.class, KSMetaData.optsWithRF(1), - SchemaLoader.standardCFMD(KEYSPACE1, CF_STANDARD1), - SchemaLoader.standardCFMD(KEYSPACE1, CF_STANDARD2)); + SchemaLoader.standardCFMD(KEYSPACE1, TABLE1), + SchemaLoader.standardCFMD(KEYSPACE1, TABLE2)); SchemaLoader.createKeyspace(KEYSPACE3, true, false, SimpleStrategy.class, KSMetaData.optsWithRF(5), - SchemaLoader.standardCFMD(KEYSPACE1, CF_STANDARD1), - SchemaLoader.indexCFMD(KEYSPACE3, CF_INDEXED, true)); + SchemaLoader.standardCFMD(KEYSPACE1, TABLE1), + SchemaLoader.compositeIndexCFMD(KEYSPACE3, TABLE1i, true)); SchemaLoader.createKeyspace(KEYSPACE6, SimpleStrategy.class, KSMetaData.optsWithRF(1), - SchemaLoader.indexCFMD(KEYSPACE6, CF_INDEXED, true)); + SchemaLoader.compositeIndexCFMD(KEYSPACE6, TABLE1i, true)); } @Test public void testCFMetaDataApply() throws ConfigurationException { - CFMetaData cfm = new CFMetaData(KEYSPACE1, - "TestApplyCFM_CF", - ColumnFamilyType.Standard, - new SimpleDenseCellNameType(BytesType.instance)); + CFMetaData cfm = CFMetaData.Builder.create(KEYSPACE1, "TestApplyCFM_CF") + .addPartitionKey("keys", BytesType.instance) + .addClusteringColumn("col", BytesType.instance).build(); + for (int i = 0; i < 5; i++) { @@ -103,7 +114,7 @@ public class DefsTest .maxCompactionThreshold(500); // we'll be adding this one later. make sure it's not already there. - Assert.assertNull(cfm.getColumnDefinition(ByteBuffer.wrap(new byte[] { 5 }))); + assertNull(cfm.getColumnDefinition(ByteBuffer.wrap(new byte[]{ 5 }))); CFMetaData cfNew = cfm.copy(); @@ -115,14 +126,14 @@ public class DefsTest // remove one. ColumnDefinition removeIndexDef = ColumnDefinition.regularDef(cfm, ByteBuffer.wrap(new byte[] { 0 }), BytesType.instance, null) .setIndex("0", IndexType.KEYS, null); - Assert.assertTrue(cfNew.removeColumnDefinition(removeIndexDef)); + assertTrue(cfNew.removeColumnDefinition(removeIndexDef)); cfm.apply(cfNew); for (int i = 1; i < cfm.allColumns().size(); i++) - Assert.assertNotNull(cfm.getColumnDefinition(ByteBuffer.wrap(new byte[] { 1 }))); - Assert.assertNull(cfm.getColumnDefinition(ByteBuffer.wrap(new byte[] { 0 }))); - Assert.assertNotNull(cfm.getColumnDefinition(ByteBuffer.wrap(new byte[] { 5 }))); + assertNotNull(cfm.getColumnDefinition(ByteBuffer.wrap(new byte[]{ 1 }))); + assertNull(cfm.getColumnDefinition(ByteBuffer.wrap(new byte[]{ 0 }))); + assertNotNull(cfm.getColumnDefinition(ByteBuffer.wrap(new byte[]{ 5 }))); } @Test @@ -130,37 +141,17 @@ public class DefsTest { String[] valid = {"1", "a", "_1", "b_", "__", "1_a"}; for (String s : valid) - Assert.assertTrue(CFMetaData.isNameValid(s)); + assertTrue(CFMetaData.isNameValid(s)); String[] invalid = {"b@t", "dash-y", "", " ", "dot.s", ".hidden"}; for (String s : invalid) - Assert.assertFalse(CFMetaData.isNameValid(s)); - } - - @Ignore - @Test - public void saveAndRestore() - { - /* - // verify dump and reload. - UUID first = UUIDGen.makeType1UUIDFromHost(FBUtilities.getBroadcastAddress()); - DefsTables.dumpToStorage(first); - List defs = new ArrayList(DefsTables.loadFromStorage(first)); - - Assert.assertTrue(defs.size() > 0); - Assert.assertEquals(defs.size(), Schema.instance.getNonSystemKeyspaces().size()); - for (KSMetaData loaded : defs) - { - KSMetaData defined = Schema.instance.getKeyspaceDefinition(loaded.name); - Assert.assertTrue(String.format("%s != %s", loaded, defined), defined.equals(loaded)); - } - */ + assertFalse(CFMetaData.isNameValid(s)); } @Test public void addNewCfToBogusKeyspace() { - CFMetaData newCf = addTestCF("MadeUpKeyspace", "NewCF", "new cf"); + CFMetaData newCf = addTestTable("MadeUpKeyspace", "NewCF", "new cf"); try { MigrationManager.announceNewColumnFamily(newCf); @@ -178,83 +169,81 @@ public class DefsTest final String cf = "BrandNewCfWithNull"; KSMetaData original = Schema.instance.getKSMetaData(ks); - CFMetaData newCf = addTestCF(original.name, cf, null); + CFMetaData newCf = addTestTable(original.name, cf, null); - Assert.assertFalse(Schema.instance.getKSMetaData(ks).cfMetaData().containsKey(newCf.cfName)); + assertFalse(Schema.instance.getKSMetaData(ks).cfMetaData().containsKey(newCf.cfName)); MigrationManager.announceNewColumnFamily(newCf); - Assert.assertTrue(Schema.instance.getKSMetaData(ks).cfMetaData().containsKey(newCf.cfName)); - Assert.assertEquals(newCf, Schema.instance.getKSMetaData(ks).cfMetaData().get(newCf.cfName)); + assertTrue(Schema.instance.getKSMetaData(ks).cfMetaData().containsKey(newCf.cfName)); + assertEquals(newCf, Schema.instance.getKSMetaData(ks).cfMetaData().get(newCf.cfName)); } @Test - public void addNewCF() throws ConfigurationException + public void addNewTable() throws ConfigurationException { - final String ks = KEYSPACE1; - final String cf = "BrandNewCf"; - KSMetaData original = Schema.instance.getKSMetaData(ks); + final String ksName = KEYSPACE1; + final String tableName = "anewtable"; + KSMetaData original = Schema.instance.getKSMetaData(ksName); - CFMetaData newCf = addTestCF(original.name, cf, "A New Table"); + CFMetaData cfm = addTestTable(original.name, tableName, "A New Table"); - Assert.assertFalse(Schema.instance.getKSMetaData(ks).cfMetaData().containsKey(newCf.cfName)); - MigrationManager.announceNewColumnFamily(newCf); + assertFalse(Schema.instance.getKSMetaData(ksName).cfMetaData().containsKey(cfm.cfName)); + MigrationManager.announceNewColumnFamily(cfm); - Assert.assertTrue(Schema.instance.getKSMetaData(ks).cfMetaData().containsKey(newCf.cfName)); - Assert.assertEquals(newCf, Schema.instance.getKSMetaData(ks).cfMetaData().get(newCf.cfName)); + assertTrue(Schema.instance.getKSMetaData(ksName).cfMetaData().containsKey(cfm.cfName)); + assertEquals(cfm, Schema.instance.getKSMetaData(ksName).cfMetaData().get(cfm.cfName)); // now read and write to it. - CellName col0 = cellname("col0"); - DecoratedKey dk = Util.dk("key0"); - Mutation rm = new Mutation(ks, dk.getKey()); - rm.add(cf, col0, ByteBufferUtil.bytes("value0"), 1L); - rm.applyUnsafe(); - ColumnFamilyStore store = Keyspace.open(ks).getColumnFamilyStore(cf); - Assert.assertNotNull(store); - store.forceBlockingFlush(); + QueryProcessor.executeInternal(String.format("INSERT INTO %s.%s (key, col, val) VALUES (?, ?, ?)", + ksName, tableName), + "key0", "col0", "val0"); - ColumnFamily cfam = store.getColumnFamily(Util.namesQueryFilter(store, dk, col0)); - Assert.assertNotNull(cfam.getColumn(col0)); - Cell col = cfam.getColumn(col0); - Assert.assertEquals(ByteBufferUtil.bytes("value0"), col.value()); + // flush to exercise more than just hitting the memtable + ColumnFamilyStore cfs = Keyspace.open(ksName).getColumnFamilyStore(tableName); + assertNotNull(cfs); + cfs.forceBlockingFlush(); + + // and make sure we get out what we put in + UntypedResultSet rows = QueryProcessor.executeInternal(String.format("SELECT * FROM %s.%s", ksName, tableName)); + assertRows(rows, row("key0", "col0", "val0")); } @Test public void dropCf() throws ConfigurationException { - DecoratedKey dk = Util.dk("dropCf"); // sanity final KSMetaData ks = Schema.instance.getKSMetaData(KEYSPACE1); - Assert.assertNotNull(ks); - final CFMetaData cfm = ks.cfMetaData().get("Standard1"); - Assert.assertNotNull(cfm); + assertNotNull(ks); + final CFMetaData cfm = ks.cfMetaData().get(TABLE1); + assertNotNull(cfm); // write some data, force a flush, then verify that files exist on disk. - Mutation rm = new Mutation(ks.name, dk.getKey()); for (int i = 0; i < 100; i++) - rm.add(cfm.cfName, cellname("col" + i), ByteBufferUtil.bytes("anyvalue"), 1L); - rm.applyUnsafe(); + QueryProcessor.executeInternal(String.format("INSERT INTO %s.%s (key, name, val) VALUES (?, ?, ?)", + KEYSPACE1, TABLE1), + "dropCf", "col" + i, "anyvalue"); ColumnFamilyStore store = Keyspace.open(cfm.ksName).getColumnFamilyStore(cfm.cfName); - Assert.assertNotNull(store); + assertNotNull(store); store.forceBlockingFlush(); - Assert.assertTrue(store.directories.sstableLister().list().size() > 0); + assertTrue(store.directories.sstableLister().list().size() > 0); MigrationManager.announceColumnFamilyDrop(ks.name, cfm.cfName); - Assert.assertFalse(Schema.instance.getKSMetaData(ks.name).cfMetaData().containsKey(cfm.cfName)); + assertFalse(Schema.instance.getKSMetaData(ks.name).cfMetaData().containsKey(cfm.cfName)); // any write should fail. - rm = new Mutation(ks.name, dk.getKey()); boolean success = true; try { - rm.add("Standard1", cellname("col0"), ByteBufferUtil.bytes("value0"), 1L); - rm.applyUnsafe(); + QueryProcessor.executeInternal(String.format("INSERT INTO %s.%s (key, name, val) VALUES (?, ?, ?)", + KEYSPACE1, TABLE1), + "dropCf", "col0", "anyvalue"); } catch (Throwable th) { success = false; } - Assert.assertFalse("This mutation should have failed since the CF no longer exists.", success); + assertFalse("This mutation should have failed since the CF no longer exists.", success); // verify that the files are gone. Supplier lambda = () -> { @@ -272,68 +261,60 @@ public class DefsTest @Test public void addNewKS() throws ConfigurationException { - DecoratedKey dk = Util.dk("key0"); - CFMetaData newCf = addTestCF("NewKeyspace1", "AddedStandard1", "A new cf for a new ks"); - - KSMetaData newKs = KSMetaData.testMetadata(newCf.ksName, SimpleStrategy.class, KSMetaData.optsWithRF(5), newCf); - + CFMetaData cfm = addTestTable("newkeyspace1", "newstandard1", "A new cf for a new ks"); + KSMetaData newKs = KSMetaData.testMetadata(cfm.ksName, SimpleStrategy.class, KSMetaData.optsWithRF(5), cfm); MigrationManager.announceNewKeyspace(newKs); - Assert.assertNotNull(Schema.instance.getKSMetaData(newCf.ksName)); - Assert.assertEquals(Schema.instance.getKSMetaData(newCf.ksName), newKs); + assertNotNull(Schema.instance.getKSMetaData(cfm.ksName)); + assertEquals(Schema.instance.getKSMetaData(cfm.ksName), newKs); // test reads and writes. - CellName col0 = cellname("col0"); - Mutation rm = new Mutation(newCf.ksName, dk.getKey()); - rm.add(newCf.cfName, col0, ByteBufferUtil.bytes("value0"), 1L); - rm.applyUnsafe(); - ColumnFamilyStore store = Keyspace.open(newCf.ksName).getColumnFamilyStore(newCf.cfName); - Assert.assertNotNull(store); + QueryProcessor.executeInternal("INSERT INTO newkeyspace1.newstandard1 (key, col, val) VALUES (?, ?, ?)", + "key0", "col0", "val0"); + ColumnFamilyStore store = Keyspace.open(cfm.ksName).getColumnFamilyStore(cfm.cfName); + assertNotNull(store); store.forceBlockingFlush(); - ColumnFamily cfam = store.getColumnFamily(Util.namesQueryFilter(store, dk, col0)); - Assert.assertNotNull(cfam.getColumn(col0)); - Cell col = cfam.getColumn(col0); - Assert.assertEquals(ByteBufferUtil.bytes("value0"), col.value()); + UntypedResultSet rows = QueryProcessor.executeInternal("SELECT * FROM newkeyspace1.newstandard1"); + assertRows(rows, row("key0", "col0", "val0")); } @Test public void dropKS() throws ConfigurationException { - DecoratedKey dk = Util.dk("dropKs"); // sanity final KSMetaData ks = Schema.instance.getKSMetaData(KEYSPACE1); - Assert.assertNotNull(ks); - final CFMetaData cfm = ks.cfMetaData().get("Standard2"); - Assert.assertNotNull(cfm); + assertNotNull(ks); + final CFMetaData cfm = ks.cfMetaData().get(TABLE2); + assertNotNull(cfm); // write some data, force a flush, then verify that files exist on disk. - Mutation rm = new Mutation(ks.name, dk.getKey()); for (int i = 0; i < 100; i++) - rm.add(cfm.cfName, cellname("col" + i), ByteBufferUtil.bytes("anyvalue"), 1L); - rm.applyUnsafe(); - ColumnFamilyStore store = Keyspace.open(cfm.ksName).getColumnFamilyStore(cfm.cfName); - Assert.assertNotNull(store); - store.forceBlockingFlush(); - Assert.assertTrue(store.directories.sstableLister().list().size() > 0); + QueryProcessor.executeInternal(String.format("INSERT INTO %s.%s (key, name, val) VALUES (?, ?, ?)", + KEYSPACE1, TABLE2), + "dropKs", "col" + i, "anyvalue"); + ColumnFamilyStore cfs = Keyspace.open(cfm.ksName).getColumnFamilyStore(cfm.cfName); + assertNotNull(cfs); + cfs.forceBlockingFlush(); + assertTrue(!cfs.directories.sstableLister().list().isEmpty()); MigrationManager.announceKeyspaceDrop(ks.name); - Assert.assertNull(Schema.instance.getKSMetaData(ks.name)); + assertNull(Schema.instance.getKSMetaData(ks.name)); // write should fail. - rm = new Mutation(ks.name, dk.getKey()); boolean success = true; try { - rm.add("Standard1", cellname("col0"), ByteBufferUtil.bytes("value0"), 1L); - rm.applyUnsafe(); + QueryProcessor.executeInternal(String.format("INSERT INTO %s.%s (key, name, val) VALUES (?, ?, ?)", + KEYSPACE1, TABLE2), + "dropKs", "col0", "anyvalue"); } catch (Throwable th) { success = false; } - Assert.assertFalse("This mutation should have failed since the CF no longer exists.", success); + assertFalse("This mutation should have failed since the KS no longer exists.", success); // reads should fail too. boolean threw = false; @@ -345,78 +326,73 @@ public class DefsTest { threw = true; } - Assert.assertTrue(threw); + assertTrue(threw); } @Test public void dropKSUnflushed() throws ConfigurationException { - DecoratedKey dk = Util.dk("dropKs"); // sanity final KSMetaData ks = Schema.instance.getKSMetaData(KEYSPACE3); - Assert.assertNotNull(ks); - final CFMetaData cfm = ks.cfMetaData().get("Standard1"); - Assert.assertNotNull(cfm); + assertNotNull(ks); + final CFMetaData cfm = ks.cfMetaData().get(TABLE1); + assertNotNull(cfm); // write some data - Mutation rm = new Mutation(ks.name, dk.getKey()); for (int i = 0; i < 100; i++) - rm.add(cfm.cfName, cellname("col" + i), ByteBufferUtil.bytes("anyvalue"), 1L); - rm.applyUnsafe(); + QueryProcessor.executeInternal(String.format("INSERT INTO %s.%s (key, name, val) VALUES (?, ?, ?)", + KEYSPACE3, TABLE1), + "dropKs", "col" + i, "anyvalue"); MigrationManager.announceKeyspaceDrop(ks.name); - Assert.assertNull(Schema.instance.getKSMetaData(ks.name)); + assertNull(Schema.instance.getKSMetaData(ks.name)); } @Test public void createEmptyKsAddNewCf() throws ConfigurationException { - Assert.assertNull(Schema.instance.getKSMetaData(EMPTYKEYSPACE)); - - KSMetaData newKs = KSMetaData.testMetadata(EMPTYKEYSPACE, SimpleStrategy.class, KSMetaData.optsWithRF(5)); - + assertNull(Schema.instance.getKSMetaData(EMPTY_KEYSPACE)); + KSMetaData newKs = KSMetaData.testMetadata(EMPTY_KEYSPACE, SimpleStrategy.class, KSMetaData.optsWithRF(5)); MigrationManager.announceNewKeyspace(newKs); - Assert.assertNotNull(Schema.instance.getKSMetaData(EMPTYKEYSPACE)); + assertNotNull(Schema.instance.getKSMetaData(EMPTY_KEYSPACE)); - CFMetaData newCf = addTestCF(EMPTYKEYSPACE, "AddedLater", "A new CF to add to an empty KS"); + String tableName = "added_later"; + CFMetaData newCf = addTestTable(EMPTY_KEYSPACE, tableName, "A new CF to add to an empty KS"); //should not exist until apply - Assert.assertFalse(Schema.instance.getKSMetaData(newKs.name).cfMetaData().containsKey(newCf.cfName)); + assertFalse(Schema.instance.getKSMetaData(newKs.name).cfMetaData().containsKey(newCf.cfName)); //add the new CF to the empty space MigrationManager.announceNewColumnFamily(newCf); - Assert.assertTrue(Schema.instance.getKSMetaData(newKs.name).cfMetaData().containsKey(newCf.cfName)); - Assert.assertEquals(Schema.instance.getKSMetaData(newKs.name).cfMetaData().get(newCf.cfName), newCf); + assertTrue(Schema.instance.getKSMetaData(newKs.name).cfMetaData().containsKey(newCf.cfName)); + assertEquals(Schema.instance.getKSMetaData(newKs.name).cfMetaData().get(newCf.cfName), newCf); // now read and write to it. - CellName col0 = cellname("col0"); - DecoratedKey dk = Util.dk("key0"); - Mutation rm = new Mutation(newKs.name, dk.getKey()); - rm.add(newCf.cfName, col0, ByteBufferUtil.bytes("value0"), 1L); - rm.applyUnsafe(); - ColumnFamilyStore store = Keyspace.open(newKs.name).getColumnFamilyStore(newCf.cfName); - Assert.assertNotNull(store); - store.forceBlockingFlush(); + QueryProcessor.executeInternal(String.format("INSERT INTO %s.%s (key, col, val) VALUES (?, ?, ?)", + EMPTY_KEYSPACE, tableName), + "key0", "col0", "val0"); - ColumnFamily cfam = store.getColumnFamily(Util.namesQueryFilter(store, dk, col0)); - Assert.assertNotNull(cfam.getColumn(col0)); - Cell col = cfam.getColumn(col0); - Assert.assertEquals(ByteBufferUtil.bytes("value0"), col.value()); + ColumnFamilyStore cfs = Keyspace.open(newKs.name).getColumnFamilyStore(newCf.cfName); + assertNotNull(cfs); + cfs.forceBlockingFlush(); + + UntypedResultSet rows = QueryProcessor.executeInternal(String.format("SELECT * FROM %s.%s", EMPTY_KEYSPACE, tableName)); + assertRows(rows, row("key0", "col0", "val0")); } @Test public void testUpdateKeyspace() throws ConfigurationException { // create a keyspace to serve as existing. - CFMetaData cf = addTestCF("UpdatedKeyspace", "AddedStandard1", "A new cf for a new ks"); + CFMetaData cf = addTestTable("UpdatedKeyspace", "AddedStandard1", "A new cf for a new ks"); KSMetaData oldKs = KSMetaData.testMetadata(cf.ksName, SimpleStrategy.class, KSMetaData.optsWithRF(5), cf); MigrationManager.announceNewKeyspace(oldKs); - Assert.assertNotNull(Schema.instance.getKSMetaData(cf.ksName)); - Assert.assertEquals(Schema.instance.getKSMetaData(cf.ksName), oldKs); + assertNotNull(Schema.instance.getKSMetaData(cf.ksName)); + assertEquals(Schema.instance.getKSMetaData(cf.ksName), oldKs); // names should match. KSMetaData newBadKs2 = KSMetaData.testMetadata(cf.ksName + "trash", SimpleStrategy.class, KSMetaData.optsWithRF(4)); @@ -434,21 +410,22 @@ public class DefsTest MigrationManager.announceKeyspaceUpdate(newKs); KSMetaData newFetchedKs = Schema.instance.getKSMetaData(newKs.name); - Assert.assertEquals(newFetchedKs.strategyClass, newKs.strategyClass); - Assert.assertFalse(newFetchedKs.strategyClass.equals(oldKs.strategyClass)); + assertEquals(newFetchedKs.strategyClass, newKs.strategyClass); + assertFalse(newFetchedKs.strategyClass.equals(oldKs.strategyClass)); } + /* @Test public void testUpdateColumnFamilyNoIndexes() throws ConfigurationException { // create a keyspace with a cf to update. - CFMetaData cf = addTestCF("UpdatedCfKs", "Standard1added", "A new cf that will be updated"); + CFMetaData cf = addTestTable("UpdatedCfKs", "Standard1added", "A new cf that will be updated"); KSMetaData ksm = KSMetaData.testMetadata(cf.ksName, SimpleStrategy.class, KSMetaData.optsWithRF(1), cf); MigrationManager.announceNewKeyspace(ksm); - Assert.assertNotNull(Schema.instance.getKSMetaData(cf.ksName)); - Assert.assertEquals(Schema.instance.getKSMetaData(cf.ksName), ksm); - Assert.assertNotNull(Schema.instance.getCFMetaData(cf.ksName, cf.cfName)); + assertNotNull(Schema.instance.getKSMetaData(cf.ksName)); + assertEquals(Schema.instance.getKSMetaData(cf.ksName), ksm); + assertNotNull(Schema.instance.getCFMetaData(cf.ksName, cf.cfName)); // updating certain fields should fail. CFMetaData newCfm = cf.copy(); @@ -478,10 +455,10 @@ public class DefsTest // can't test changing the reconciler because there is only one impl. // check the cumulative affect. - Assert.assertEquals(Schema.instance.getCFMetaData(cf.ksName, cf.cfName).getComment(), newCfm.getComment()); - Assert.assertEquals(Schema.instance.getCFMetaData(cf.ksName, cf.cfName).getReadRepairChance(), newCfm.getReadRepairChance(), 0.0001); - Assert.assertEquals(Schema.instance.getCFMetaData(cf.ksName, cf.cfName).getGcGraceSeconds(), newCfm.getGcGraceSeconds()); - Assert.assertEquals(UTF8Type.instance, Schema.instance.getCFMetaData(cf.ksName, cf.cfName).getDefaultValidator()); + assertEquals(Schema.instance.getCFMetaData(cf.ksName, cf.cfName).getComment(), newCfm.getComment()); + assertEquals(Schema.instance.getCFMetaData(cf.ksName, cf.cfName).getReadRepairChance(), newCfm.getReadRepairChance(), 0.0001); + assertEquals(Schema.instance.getCFMetaData(cf.ksName, cf.cfName).getGcGraceSeconds(), newCfm.getGcGraceSeconds()); + assertEquals(UTF8Type.instance, Schema.instance.getCFMetaData(cf.ksName, cf.cfName).getDefaultValidator()); // Change cfId newCfm = new CFMetaData(cf.ksName, cf.cfName, cf.cfType, cf.comparator); @@ -533,39 +510,44 @@ public class DefsTest } catch (ConfigurationException expected) {} } + */ @Test public void testDropIndex() throws ConfigurationException { // persist keyspace definition in the system keyspace LegacySchemaTables.makeCreateKeyspaceMutation(Schema.instance.getKSMetaData(KEYSPACE6), FBUtilities.timestampMicros()).applyUnsafe(); - ColumnFamilyStore cfs = Keyspace.open(KEYSPACE6).getColumnFamilyStore("Indexed1"); + ColumnFamilyStore cfs = Keyspace.open(KEYSPACE6).getColumnFamilyStore(TABLE1i); // insert some data. save the sstable descriptor so we can make sure it's marked for delete after the drop - Mutation rm = new Mutation(KEYSPACE6, ByteBufferUtil.bytes("k1")); - rm.add("Indexed1", cellname("notbirthdate"), ByteBufferUtil.bytes(1L), 0); - rm.add("Indexed1", cellname("birthdate"), ByteBufferUtil.bytes(1L), 0); - rm.applyUnsafe(); + QueryProcessor.executeInternal(String.format("INSERT INTO %s.%s (key, c1, birthdate, notbirthdate) VALUES (?, ?, ?, ?)", + KEYSPACE6, TABLE1i), + "key0", "col0", 1L, 1L); + cfs.forceBlockingFlush(); - ColumnFamilyStore indexedCfs = cfs.indexManager.getIndexForColumn(ByteBufferUtil.bytes("birthdate")).getIndexCfs(); + ColumnFamilyStore indexedCfs = cfs.indexManager.getIndexForColumn(cfs.metadata.getColumnDefinition(ByteBufferUtil.bytes("birthdate"))).getIndexCfs(); Descriptor desc = indexedCfs.getSSTables().iterator().next().descriptor; // drop the index CFMetaData meta = cfs.metadata.copy(); - ColumnDefinition cdOld = meta.regularColumns().iterator().next(); + ColumnDefinition cdOld = cfs.metadata.getColumnDefinition(ByteBufferUtil.bytes("birthdate")); ColumnDefinition cdNew = ColumnDefinition.regularDef(meta, cdOld.name.bytes, cdOld.type, null); meta.addOrReplaceColumnDefinition(cdNew); MigrationManager.announceColumnFamilyUpdate(meta, false); // check - Assert.assertTrue(cfs.indexManager.getIndexes().isEmpty()); + assertTrue(cfs.indexManager.getIndexes().isEmpty()); SSTableDeletingTask.waitForDeletions(); - Assert.assertFalse(new File(desc.filenameFor(Component.DATA)).exists()); + assertFalse(new File(desc.filenameFor(Component.DATA)).exists()); } - private CFMetaData addTestCF(String ks, String cf, String comment) + private CFMetaData addTestTable(String ks, String cf, String comment) { - CFMetaData newCFMD = new CFMetaData(ks, cf, ColumnFamilyType.Standard, new SimpleDenseCellNameType(UTF8Type.instance)); + CFMetaData newCFMD = CFMetaData.Builder.create(ks, cf) + .addPartitionKey("key", UTF8Type.instance) + .addClusteringColumn("col", UTF8Type.instance) + .addRegularColumn("val", UTF8Type.instance).build(); + newCFMD.comment(comment) .readRepairChance(0.0); diff --git a/test/unit/org/apache/cassandra/service/ActiveRepairServiceTest.java b/test/unit/org/apache/cassandra/service/ActiveRepairServiceTest.java index dab45f91cf..25d4134485 100644 --- a/test/unit/org/apache/cassandra/service/ActiveRepairServiceTest.java +++ b/test/unit/org/apache/cassandra/service/ActiveRepairServiceTest.java @@ -58,10 +58,10 @@ public class ActiveRepairServiceTest { SchemaLoader.prepareServer(); SchemaLoader.createKeyspace(KEYSPACE5, - SimpleStrategy.class, - KSMetaData.optsWithRF(2), - SchemaLoader.standardCFMD(KEYSPACE5, CF_COUNTER), - SchemaLoader.standardCFMD(KEYSPACE5, CF_STANDRAD1)); + SimpleStrategy.class, + KSMetaData.optsWithRF(2), + SchemaLoader.standardCFMD(KEYSPACE5, CF_COUNTER), + SchemaLoader.standardCFMD(KEYSPACE5, CF_STANDRAD1)); } @Before @@ -190,8 +190,8 @@ public class ActiveRepairServiceTest Collection hosts = Arrays.asList(FBUtilities.getBroadcastAddress().getCanonicalHostName(),expected.get(0).getCanonicalHostName()); assertEquals(expected.get(0), ActiveRepairService.getNeighbors(KEYSPACE5, - StorageService.instance.getLocalRanges(KEYSPACE5).iterator().next(), - null, hosts).iterator().next()); + StorageService.instance.getLocalRanges(KEYSPACE5).iterator().next(), + null, hosts).iterator().next()); } @Test(expected = IllegalArgumentException.class) diff --git a/test/unit/org/apache/cassandra/service/BatchlogEndpointFilterTest.java b/test/unit/org/apache/cassandra/service/BatchlogEndpointFilterTest.java index 72e8df5d06..186cc41384 100644 --- a/test/unit/org/apache/cassandra/service/BatchlogEndpointFilterTest.java +++ b/test/unit/org/apache/cassandra/service/BatchlogEndpointFilterTest.java @@ -20,16 +20,16 @@ package org.apache.cassandra.service; import java.net.InetAddress; import java.net.UnknownHostException; import java.util.Collection; -import org.junit.Test; -import org.junit.matchers.JUnitMatchers; import com.google.common.collect.ImmutableMultimap; import com.google.common.collect.Multimap; +import org.junit.Test; +import org.junit.matchers.JUnitMatchers; import org.apache.cassandra.db.BatchlogManager; -import static org.junit.Assert.assertThat; import static org.hamcrest.CoreMatchers.is; +import static org.junit.Assert.assertThat; public class BatchlogEndpointFilterTest { diff --git a/test/unit/org/apache/cassandra/service/DataResolverTest.java b/test/unit/org/apache/cassandra/service/DataResolverTest.java new file mode 100644 index 0000000000..b4902a7126 --- /dev/null +++ b/test/unit/org/apache/cassandra/service/DataResolverTest.java @@ -0,0 +1,535 @@ +package org.apache.cassandra.service; +/* + * + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + * + */ + + +import java.net.InetAddress; +import java.net.UnknownHostException; +import java.util.*; + +import com.google.common.collect.Iterators; +import org.junit.*; + +import org.apache.cassandra.SchemaLoader; +import org.apache.cassandra.Util; +import org.apache.cassandra.config.CFMetaData; +import org.apache.cassandra.config.KSMetaData; +import org.apache.cassandra.db.*; +import org.apache.cassandra.db.rows.*; +import org.apache.cassandra.db.marshal.AsciiType; +import org.apache.cassandra.db.marshal.BytesType; +import org.apache.cassandra.db.partitions.*; +import org.apache.cassandra.exceptions.ConfigurationException; +import org.apache.cassandra.locator.SimpleStrategy; +import org.apache.cassandra.net.*; +import org.apache.cassandra.utils.FBUtilities; + +import static org.apache.cassandra.Util.assertClustering; +import static org.apache.cassandra.Util.assertColumn; +import static org.apache.cassandra.Util.assertColumns; +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertFalse; +import static org.junit.Assert.assertNotNull; +import static org.junit.Assert.assertTrue; + +public class DataResolverTest +{ + public static final String KEYSPACE1 = "DataResolverTest"; + public static final String CF_STANDARD = "Standard1"; + + // counter to generate the last byte of the respondent's address in a ReadResponse message + private int addressSuffix = 10; + + private DecoratedKey dk; + private Keyspace ks; + private ColumnFamilyStore cfs; + private CFMetaData cfm; + private int nowInSec; + private ReadCommand command; + private MessageRecorder messageRecorder; + + + @BeforeClass + public static void defineSchema() throws ConfigurationException + { + CFMetaData cfMetadata = CFMetaData.Builder.create(KEYSPACE1, CF_STANDARD) + .addPartitionKey("key", BytesType.instance) + .addClusteringColumn("col1", AsciiType.instance) + .addRegularColumn("c1", AsciiType.instance) + .addRegularColumn("c2", AsciiType.instance) + .addRegularColumn("one", AsciiType.instance) + .addRegularColumn("two", AsciiType.instance) + .build(); + SchemaLoader.prepareServer(); + SchemaLoader.createKeyspace(KEYSPACE1, + SimpleStrategy.class, + KSMetaData.optsWithRF(1), + cfMetadata); + } + + @Before + public void setup() + { + dk = Util.dk("key1"); + ks = Keyspace.open(KEYSPACE1); + cfs = ks.getColumnFamilyStore(CF_STANDARD); + cfm = cfs.metadata; + nowInSec = FBUtilities.nowInSeconds(); + command = Util.cmd(cfs, dk).withNowInSeconds(nowInSec).build(); + } + + @Before + public void injectMessageSink() + { + // install an IMessageSink to capture all messages + // so we can inspect them during tests + messageRecorder = new MessageRecorder(); + MessagingService.instance().addMessageSink(messageRecorder); + } + + @After + public void removeMessageSink() + { + // should be unnecessary, but good housekeeping + MessagingService.instance().clearMessageSinks(); + } + + @Test + public void testResolveNewerSingleRow() throws UnknownHostException + { + DataResolver resolver = new DataResolver(ks, command, ConsistencyLevel.ALL, 2); + InetAddress peer1 = peer(); + resolver.preprocess(readResponseMessage(peer1, iter(new RowUpdateBuilder(cfm, nowInSec, 0L, dk).clustering("1") + .add("c1", "v1") + .buildUpdate()))); + InetAddress peer2 = peer(); + resolver.preprocess(readResponseMessage(peer2, iter(new RowUpdateBuilder(cfm, nowInSec, 1L, dk).clustering("1") + .add("c1", "v2") + .buildUpdate()))); + + try(PartitionIterator data = resolver.resolve(); + RowIterator rows = Iterators.getOnlyElement(data)) + { + Row row = Iterators.getOnlyElement(rows); + assertColumns(row, "c1"); + assertColumn(cfm, row, "c1", "v2", 1); + } + + assertEquals(1, messageRecorder.sent.size()); + // peer 1 just needs to repair with the row from peer 2 + MessageOut msg = getSentMessage(peer1); + assertRepairMetadata(msg); + assertRepairContainsNoDeletions(msg); + assertRepairContainsColumn(msg, "1", "c1", "v2", 1); + } + + @Test + public void testResolveDisjointSingleRow() + { + DataResolver resolver = new DataResolver(ks, command, ConsistencyLevel.ALL, 2); + InetAddress peer1 = peer(); + resolver.preprocess(readResponseMessage(peer1, iter(new RowUpdateBuilder(cfm, nowInSec, 0L, dk).clustering("1") + .add("c1", "v1") + .buildUpdate()))); + + InetAddress peer2 = peer(); + resolver.preprocess(readResponseMessage(peer2, iter(new RowUpdateBuilder(cfm, nowInSec, 1L, dk).clustering("1") + .add("c2", "v2") + .buildUpdate()))); + + try(PartitionIterator data = resolver.resolve(); + RowIterator rows = Iterators.getOnlyElement(data)) + { + Row row = Iterators.getOnlyElement(rows); + assertColumns(row, "c1", "c2"); + assertColumn(cfm, row, "c1", "v1", 0); + assertColumn(cfm, row, "c2", "v2", 1); + } + + assertEquals(2, messageRecorder.sent.size()); + // each peer needs to repair with each other's column + MessageOut msg = getSentMessage(peer1); + assertRepairMetadata(msg); + assertRepairContainsColumn(msg, "1", "c2", "v2", 1); + + msg = getSentMessage(peer2); + assertRepairMetadata(msg); + assertRepairContainsColumn(msg, "1", "c1", "v1", 0); + } + + @Test + public void testResolveDisjointMultipleRows() throws UnknownHostException + { + + DataResolver resolver = new DataResolver(ks, command, ConsistencyLevel.ALL, 2); + InetAddress peer1 = peer(); + resolver.preprocess(readResponseMessage(peer1, iter(new RowUpdateBuilder(cfm, nowInSec, 0L, dk).clustering("1") + .add("c1", "v1") + .buildUpdate()))); + InetAddress peer2 = peer(); + resolver.preprocess(readResponseMessage(peer2, iter(new RowUpdateBuilder(cfm, nowInSec, 1L, dk).clustering("2") + .add("c2", "v2") + .buildUpdate()))); + + try (PartitionIterator data = resolver.resolve()) + { + try (RowIterator rows = data.next()) + { + // We expect the resolved superset to contain both rows + Row row = rows.next(); + assertClustering(cfm, row, "1"); + assertColumns(row, "c1"); + assertColumn(cfm, row, "c1", "v1", 0); + + row = rows.next(); + assertClustering(cfm, row, "2"); + assertColumns(row, "c2"); + assertColumn(cfm, row, "c2", "v2", 1); + + assertFalse(rows.hasNext()); + assertFalse(data.hasNext()); + } + } + + assertEquals(2, messageRecorder.sent.size()); + // each peer needs to repair the row from the other + MessageOut msg = getSentMessage(peer1); + assertRepairMetadata(msg); + assertRepairContainsNoDeletions(msg); + assertRepairContainsColumn(msg, "2", "c2", "v2", 1); + + msg = getSentMessage(peer2); + assertRepairMetadata(msg); + assertRepairContainsNoDeletions(msg); + assertRepairContainsColumn(msg, "1", "c1", "v1", 0); + } + + @Test + public void testResolveDisjointMultipleRowsWithRangeTombstones() + { + DataResolver resolver = new DataResolver(ks, command, ConsistencyLevel.ALL, 4); + + RangeTombstone tombstone1 = tombstone("1", "11", 1, nowInSec); + RangeTombstone tombstone2 = tombstone("3", "31", 1, nowInSec); + PartitionUpdate update =new RowUpdateBuilder(cfm, nowInSec, 1L, dk).addRangeTombstone(tombstone1) + .addRangeTombstone(tombstone2) + .buildUpdate(); + + InetAddress peer1 = peer(); + UnfilteredPartitionIterator iter1 = iter(new RowUpdateBuilder(cfm, nowInSec, 1L, dk).addRangeTombstone(tombstone1) + .addRangeTombstone(tombstone2) + .buildUpdate()); + resolver.preprocess(readResponseMessage(peer1, iter1)); + // not covered by any range tombstone + InetAddress peer2 = peer(); + UnfilteredPartitionIterator iter2 = iter(new RowUpdateBuilder(cfm, nowInSec, 0L, dk).clustering("0") + .add("c1", "v0") + .buildUpdate()); + resolver.preprocess(readResponseMessage(peer2, iter2)); + // covered by a range tombstone + InetAddress peer3 = peer(); + UnfilteredPartitionIterator iter3 = iter(new RowUpdateBuilder(cfm, nowInSec, 0L, dk).clustering("10") + .add("c2", "v1") + .buildUpdate()); + resolver.preprocess(readResponseMessage(peer3, iter3)); + // range covered by rt, but newer + InetAddress peer4 = peer(); + UnfilteredPartitionIterator iter4 = iter(new RowUpdateBuilder(cfm, nowInSec, 2L, dk).clustering("3") + .add("one", "A") + .buildUpdate()); + resolver.preprocess(readResponseMessage(peer4, iter4)); + try (PartitionIterator data = resolver.resolve()) + { + try (RowIterator rows = data.next()) + { + Row row = rows.next(); + assertClustering(cfm, row, "0"); + assertColumns(row, "c1"); + assertColumn(cfm, row, "c1", "v0", 0); + + row = rows.next(); + assertClustering(cfm, row, "3"); + assertColumns(row, "one"); + assertColumn(cfm, row, "one", "A", 2); + + assertFalse(rows.hasNext()); + } + } + + assertEquals(4, messageRecorder.sent.size()); + // peer1 needs the rows from peers 2 and 4 + MessageOut msg = getSentMessage(peer1); + assertRepairMetadata(msg); + assertRepairContainsNoDeletions(msg); + assertRepairContainsColumn(msg, "0", "c1", "v0", 0); + assertRepairContainsColumn(msg, "3", "one", "A", 2); + + // peer2 needs to get the row from peer4 and the RTs + msg = getSentMessage(peer2); + assertRepairMetadata(msg); + assertRepairContainsDeletions(msg, null, tombstone1, tombstone2); + assertRepairContainsColumn(msg, "3", "one", "A", 2); + + // peer 3 needs both rows and the RTs + msg = getSentMessage(peer3); + assertRepairMetadata(msg); + assertRepairContainsDeletions(msg, null, tombstone1, tombstone2); + assertRepairContainsColumn(msg, "0", "c1", "v0", 0); + assertRepairContainsColumn(msg, "3", "one", "A", 2); + + // peer4 needs the row from peer2 and the RTs + msg = getSentMessage(peer4); + assertRepairMetadata(msg); + assertRepairContainsDeletions(msg, null, tombstone1, tombstone2); + assertRepairContainsColumn(msg, "0", "c1", "v0", 0); + } + + @Test + public void testResolveWithOneEmpty() + { + DataResolver resolver = new DataResolver(ks, command, ConsistencyLevel.ALL, 2); + InetAddress peer1 = peer(); + resolver.preprocess(readResponseMessage(peer1, iter(new RowUpdateBuilder(cfm, nowInSec, 1L, dk).clustering("1") + .add("c2", "v2") + .buildUpdate()))); + InetAddress peer2 = peer(); + resolver.preprocess(readResponseMessage(peer2, UnfilteredPartitionIterators.EMPTY)); + + try(PartitionIterator data = resolver.resolve(); + RowIterator rows = Iterators.getOnlyElement(data)) + { + Row row = Iterators.getOnlyElement(rows); + assertColumns(row, "c2"); + assertColumn(cfm, row, "c2", "v2", 1); + } + + assertEquals(1, messageRecorder.sent.size()); + // peer 2 needs the row from peer 1 + MessageOut msg = getSentMessage(peer2); + assertRepairMetadata(msg); + assertRepairContainsNoDeletions(msg); + assertRepairContainsColumn(msg, "1", "c2", "v2", 1); + } + + @Test + public void testResolveWithBothEmpty() + { + DataResolver resolver = new DataResolver(ks, command, ConsistencyLevel.ALL, 2); + resolver.preprocess(readResponseMessage(peer(), UnfilteredPartitionIterators.EMPTY)); + resolver.preprocess(readResponseMessage(peer(), UnfilteredPartitionIterators.EMPTY)); + + try(PartitionIterator data = resolver.resolve()) + { + assertFalse(data.hasNext()); + } + + assertTrue(messageRecorder.sent.isEmpty()); + } + + @Test + public void testResolveDeleted() + { + DataResolver resolver = new DataResolver(ks, command, ConsistencyLevel.ALL, 2); + // one response with columns timestamped before a delete in another response + InetAddress peer1 = peer(); + resolver.preprocess(readResponseMessage(peer1, iter(new RowUpdateBuilder(cfm, nowInSec, 0L, dk).clustering("1") + .add("one", "A") + .buildUpdate()))); + InetAddress peer2 = peer(); + resolver.preprocess(readResponseMessage(peer2, fullPartitionDelete(cfm, dk, 1, nowInSec))); + + try (PartitionIterator data = resolver.resolve()) + { + assertFalse(data.hasNext()); + } + + // peer1 should get the deletion from peer2 + assertEquals(1, messageRecorder.sent.size()); + MessageOut msg = getSentMessage(peer1); + assertRepairMetadata(msg); + assertRepairContainsDeletions(msg, new SimpleDeletionTime(1, nowInSec)); + assertRepairContainsNoColumns(msg); + } + + @Test + public void testResolveMultipleDeleted() + { + DataResolver resolver = new DataResolver(ks, command, ConsistencyLevel.ALL, 4); + // deletes and columns with interleaved timestamp, with out of order return sequence + InetAddress peer1 = peer(); + resolver.preprocess(readResponseMessage(peer1, fullPartitionDelete(cfm, dk, 0, nowInSec))); + // these columns created after the previous deletion + InetAddress peer2 = peer(); + resolver.preprocess(readResponseMessage(peer2, iter(new RowUpdateBuilder(cfm, nowInSec, 1L, dk).clustering("1") + .add("one", "A") + .add("two", "A") + .buildUpdate()))); + //this column created after the next delete + InetAddress peer3 = peer(); + resolver.preprocess(readResponseMessage(peer3, iter(new RowUpdateBuilder(cfm, nowInSec, 3L, dk).clustering("1") + .add("two", "B") + .buildUpdate()))); + InetAddress peer4 = peer(); + resolver.preprocess(readResponseMessage(peer4, fullPartitionDelete(cfm, dk, 2, nowInSec))); + + try(PartitionIterator data = resolver.resolve(); + RowIterator rows = Iterators.getOnlyElement(data)) + { + Row row = Iterators.getOnlyElement(rows); + assertColumns(row, "two"); + assertColumn(cfm, row, "two", "B", 3); + } + + // peer 1 needs to get the partition delete from peer 4 and the row from peer 3 + assertEquals(4, messageRecorder.sent.size()); + MessageOut msg = getSentMessage(peer1); + assertRepairMetadata(msg); + assertRepairContainsDeletions(msg, new SimpleDeletionTime(2, nowInSec)); + assertRepairContainsColumn(msg, "1", "two", "B", 3); + + // peer 2 needs the deletion from peer 4 and the row from peer 3 + msg = getSentMessage(peer2); + assertRepairMetadata(msg); + assertRepairContainsDeletions(msg, new SimpleDeletionTime(2, nowInSec)); + assertRepairContainsColumn(msg, "1", "two", "B", 3); + + // peer 3 needs just the deletion from peer 4 + msg = getSentMessage(peer3); + assertRepairMetadata(msg); + assertRepairContainsDeletions(msg, new SimpleDeletionTime(2, nowInSec)); + assertRepairContainsNoColumns(msg); + + // peer 4 needs just the row from peer 3 + msg = getSentMessage(peer4); + assertRepairMetadata(msg); + assertRepairContainsNoDeletions(msg); + assertRepairContainsColumn(msg, "1", "two", "B", 3); + } + + private InetAddress peer() + { + try + { + return InetAddress.getByAddress(new byte[]{ 127, 0, 0, (byte) addressSuffix++ }); + } + catch (UnknownHostException e) + { + throw new RuntimeException(e); + } + } + + private MessageOut getSentMessage(InetAddress target) + { + MessageOut message = messageRecorder.sent.get(target); + assertNotNull(String.format("No repair message was sent to %s", target), message); + return message; + } + + private void assertRepairContainsDeletions(MessageOut message, + DeletionTime deletionTime, + RangeTombstone...rangeTombstones) + { + PartitionUpdate update = ((Mutation)message.payload).getPartitionUpdates().iterator().next(); + DeletionInfo deletionInfo = update.deletionInfo(); + if (deletionTime != null) + assertEquals(deletionTime, deletionInfo.getPartitionDeletion()); + + assertEquals(rangeTombstones.length, deletionInfo.rangeCount()); + Iterator ranges = deletionInfo.rangeIterator(false); + int i = 0; + while (ranges.hasNext()) + { + assertEquals(ranges.next(), rangeTombstones[i++]); + } + } + + private void assertRepairContainsNoDeletions(MessageOut message) + { + PartitionUpdate update = ((Mutation)message.payload).getPartitionUpdates().iterator().next(); + assertTrue(update.deletionInfo().isLive()); + } + + private void assertRepairContainsColumn(MessageOut message, + String clustering, + String columnName, + String value, + long timestamp) + { + PartitionUpdate update = ((Mutation)message.payload).getPartitionUpdates().iterator().next(); + Row row = update.getRow(update.metadata().comparator.make(clustering)); + assertNotNull(row); + assertColumn(cfm, row, columnName, value, timestamp); + } + + private void assertRepairContainsNoColumns(MessageOut message) + { + PartitionUpdate update = ((Mutation)message.payload).getPartitionUpdates().iterator().next(); + assertFalse(update.iterator().hasNext()); + } + + private void assertRepairMetadata(MessageOut message) + { + assertEquals(MessagingService.Verb.READ_REPAIR, message.verb); + PartitionUpdate update = ((Mutation)message.payload).getPartitionUpdates().iterator().next(); + assertEquals(update.metadata().ksName, cfm.ksName); + assertEquals(update.metadata().cfName, cfm.cfName); + } + + public MessageIn readResponseMessage(InetAddress from, UnfilteredPartitionIterator partitionIterator) + { + return MessageIn.create(from, + ReadResponse.createDataResponse(partitionIterator), + Collections.EMPTY_MAP, + MessagingService.Verb.REQUEST_RESPONSE, + MessagingService.current_version); + } + + private RangeTombstone tombstone(Object start, Object end, long markedForDeleteAt, int localDeletionTime) + { + return new RangeTombstone(Slice.make(cfm.comparator.make(start), cfm.comparator.make(end)), + new SimpleDeletionTime(markedForDeleteAt, localDeletionTime)); + } + + private UnfilteredPartitionIterator fullPartitionDelete(CFMetaData cfm, DecoratedKey dk, long timestamp, int nowInSec) + { + return new SingletonUnfilteredPartitionIterator(PartitionUpdate.fullPartitionDelete(cfm, dk, timestamp, nowInSec).unfilteredIterator(), false); + } + + private static class MessageRecorder implements IMessageSink + { + Map sent = new HashMap<>(); + public boolean allowOutgoingMessage(MessageOut message, int id, InetAddress to) + { + sent.put(to, message); + return false; + } + + public boolean allowIncomingMessage(MessageIn message, int id) + { + return false; + } + } + + private UnfilteredPartitionIterator iter(PartitionUpdate update) + { + return new SingletonUnfilteredPartitionIterator(update.unfilteredIterator(), false); + } +} diff --git a/test/unit/org/apache/cassandra/service/EmbeddedCassandraServiceTest.java b/test/unit/org/apache/cassandra/service/EmbeddedCassandraServiceTest.java index ed0efeec01..b9367e4383 100644 --- a/test/unit/org/apache/cassandra/service/EmbeddedCassandraServiceTest.java +++ b/test/unit/org/apache/cassandra/service/EmbeddedCassandraServiceTest.java @@ -26,8 +26,10 @@ import org.junit.BeforeClass; import org.junit.Test; import org.apache.cassandra.SchemaLoader; +import org.apache.cassandra.config.CFMetaData; import org.apache.cassandra.config.DatabaseDescriptor; import org.apache.cassandra.config.KSMetaData; +import org.apache.cassandra.db.marshal.AsciiType; import org.apache.cassandra.locator.SimpleStrategy; import org.apache.cassandra.thrift.*; import org.apache.cassandra.utils.ByteBufferUtil; @@ -61,9 +63,13 @@ public class EmbeddedCassandraServiceTest SchemaLoader.prepareServer(); setup(); SchemaLoader.createKeyspace(KEYSPACE1, - SimpleStrategy.class, - KSMetaData.optsWithRF(1), - SchemaLoader.standardCFMD(KEYSPACE1, CF_STANDARD)); + SimpleStrategy.class, + KSMetaData.optsWithRF(1), + CFMetaData.Builder.create(KEYSPACE1, CF_STANDARD, true, false, false) + .addPartitionKey("pk", AsciiType.instance) + .addClusteringColumn("ck", AsciiType.instance) + .addRegularColumn("val", AsciiType.instance) + .build()); } /** diff --git a/test/unit/org/apache/cassandra/service/LeaveAndBootstrapTest.java b/test/unit/org/apache/cassandra/service/LeaveAndBootstrapTest.java index 4a09b7ad0b..c2a976904b 100644 --- a/test/unit/org/apache/cassandra/service/LeaveAndBootstrapTest.java +++ b/test/unit/org/apache/cassandra/service/LeaveAndBootstrapTest.java @@ -23,20 +23,21 @@ import java.net.InetAddress; import java.net.UnknownHostException; import java.util.*; -import org.junit.Test; - -import static org.junit.Assert.*; -import org.junit.AfterClass; -import org.junit.BeforeClass; import com.google.common.collect.HashMultimap; import com.google.common.collect.Multimap; +import org.junit.AfterClass; +import org.junit.BeforeClass; +import org.junit.Test; + import org.apache.cassandra.SchemaLoader; import org.apache.cassandra.Util; import org.apache.cassandra.config.KSMetaData; import org.apache.cassandra.config.Schema; import org.apache.cassandra.db.SystemKeyspace; -import org.apache.cassandra.dht.*; +import org.apache.cassandra.dht.IPartitioner; +import org.apache.cassandra.dht.RandomPartitioner; import org.apache.cassandra.dht.RandomPartitioner.BigIntegerToken; +import org.apache.cassandra.dht.Token; import org.apache.cassandra.gms.ApplicationState; import org.apache.cassandra.gms.Gossiper; import org.apache.cassandra.gms.VersionedValue; @@ -44,6 +45,8 @@ import org.apache.cassandra.locator.AbstractReplicationStrategy; import org.apache.cassandra.locator.SimpleSnitch; import org.apache.cassandra.locator.TokenMetadata; +import static org.junit.Assert.*; + public class LeaveAndBootstrapTest { private static final IPartitioner partitioner = RandomPartitioner.instance; @@ -691,7 +694,7 @@ public class LeaveAndBootstrapTest // create a ring of 1 node StorageService ss = StorageService.instance; VersionedValue.VersionedValueFactory valueFactory = new VersionedValue.VersionedValueFactory(partitioner); - Util.createInitialRing(ss, partitioner, new ArrayList(), new ArrayList(), new ArrayList(), new ArrayList(), 1); + Util.createInitialRing(ss, partitioner, new ArrayList(), new ArrayList(), new ArrayList(), new ArrayList(), 1); // make a REMOVING state change on a non-member endpoint; without the CASSANDRA-6564 fix, this // would result in an ArrayIndexOutOfBoundsException diff --git a/test/unit/org/apache/cassandra/service/MoveTest.java b/test/unit/org/apache/cassandra/service/MoveTest.java index 6c9e589630..cf20273747 100644 --- a/test/unit/org/apache/cassandra/service/MoveTest.java +++ b/test/unit/org/apache/cassandra/service/MoveTest.java @@ -25,27 +25,31 @@ import java.util.*; import com.google.common.collect.HashMultimap; import com.google.common.collect.Multimap; -import static org.junit.Assert.*; - -import org.apache.cassandra.gms.Gossiper; import org.junit.AfterClass; import org.junit.Before; import org.junit.BeforeClass; import org.junit.Test; -import org.apache.cassandra.exceptions.ConfigurationException; -import org.apache.cassandra.config.Schema; import org.apache.cassandra.SchemaLoader; import org.apache.cassandra.Util; import org.apache.cassandra.config.KSMetaData; -import org.apache.cassandra.dht.*; +import org.apache.cassandra.config.Schema; +import org.apache.cassandra.dht.IPartitioner; +import org.apache.cassandra.dht.RandomPartitioner; import org.apache.cassandra.dht.RandomPartitioner.BigIntegerToken; +import org.apache.cassandra.dht.Range; +import org.apache.cassandra.dht.Token; +import org.apache.cassandra.exceptions.ConfigurationException; import org.apache.cassandra.gms.ApplicationState; +import org.apache.cassandra.gms.Gossiper; import org.apache.cassandra.gms.VersionedValue; import org.apache.cassandra.locator.AbstractReplicationStrategy; import org.apache.cassandra.locator.SimpleSnitch; import org.apache.cassandra.locator.TokenMetadata; +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertTrue; + public class MoveTest { private static final IPartitioner partitioner = RandomPartitioner.instance; diff --git a/test/unit/org/apache/cassandra/service/PaxosStateTest.java b/test/unit/org/apache/cassandra/service/PaxosStateTest.java index 7f4bc49d67..9ee91ddc5b 100644 --- a/test/unit/org/apache/cassandra/service/PaxosStateTest.java +++ b/test/unit/org/apache/cassandra/service/PaxosStateTest.java @@ -19,6 +19,7 @@ package org.apache.cassandra.service; import java.nio.ByteBuffer; +import com.google.common.collect.Iterables; import org.junit.AfterClass; import org.junit.BeforeClass; import org.junit.Test; @@ -26,8 +27,9 @@ import org.junit.Test; import org.apache.cassandra.SchemaLoader; import org.apache.cassandra.Util; import org.apache.cassandra.db.*; -import org.apache.cassandra.db.composites.CellName; -import org.apache.cassandra.db.filter.QueryFilter; +import org.apache.cassandra.db.rows.UnfilteredRowIterator; +import org.apache.cassandra.db.rows.Row; +import org.apache.cassandra.db.partitions.PartitionUpdate; import org.apache.cassandra.gms.Gossiper; import org.apache.cassandra.service.paxos.Commit; import org.apache.cassandra.service.paxos.PaxosState; @@ -35,9 +37,7 @@ import org.apache.cassandra.utils.ByteBufferUtil; import org.apache.cassandra.utils.FBUtilities; import org.apache.cassandra.utils.UUIDGen; -import static org.junit.Assert.assertEquals; -import static org.junit.Assert.assertFalse; -import static org.junit.Assert.assertNull; +import static org.junit.Assert.*; public class PaxosStateTest { @@ -58,48 +58,47 @@ public class PaxosStateTest public void testCommittingAfterTruncation() throws Exception { ColumnFamilyStore cfs = Keyspace.open("PaxosStateTestKeyspace1").getColumnFamilyStore("Standard1"); - DecoratedKey key = Util.dk("key" + System.nanoTime()); - CellName name = Util.cellname("col"); + String key = "key" + System.nanoTime(); ByteBuffer value = ByteBufferUtil.bytes(0); - ColumnFamily update = ArrayBackedSortedColumns.factory.create(cfs.metadata); - update.addColumn(name, value, FBUtilities.timestampMicros()); + RowUpdateBuilder builder = new RowUpdateBuilder(cfs.metadata, FBUtilities.timestampMicros(), key); + builder.clustering("a").add("val", value); + PartitionUpdate update = Iterables.getOnlyElement(builder.build().getPartitionUpdates()); // CFS should be empty initially - assertNoDataPresent(cfs, key); + assertNoDataPresent(cfs, Util.dk(key)); // Commit the proposal & verify the data is present - Commit beforeTruncate = newProposal(0, key.getKey(), update); + Commit beforeTruncate = newProposal(0, update); PaxosState.commit(beforeTruncate); - assertDataPresent(cfs, key, name, value); + assertDataPresent(cfs, Util.dk(key), "val", value); // Truncate then attempt to commit again, mutation should // be ignored as the proposal predates the truncation cfs.truncateBlocking(); PaxosState.commit(beforeTruncate); - assertNoDataPresent(cfs, key); + assertNoDataPresent(cfs, Util.dk(key)); // Now try again with a ballot created after the truncation long timestamp = SystemKeyspace.getTruncatedAt(update.metadata().cfId) + 1; - Commit afterTruncate = newProposal(timestamp, key.getKey(), update); + Commit afterTruncate = newProposal(timestamp, update); PaxosState.commit(afterTruncate); - assertDataPresent(cfs, key, name, value); + assertDataPresent(cfs, Util.dk(key), "val", value); } - private Commit newProposal(long ballotMillis, ByteBuffer key, ColumnFamily update) + private Commit newProposal(long ballotMillis, PartitionUpdate update) { - return Commit.newProposal(key, UUIDGen.getTimeUUID(ballotMillis), update); + return Commit.newProposal(UUIDGen.getTimeUUID(ballotMillis), update); } - private void assertDataPresent(ColumnFamilyStore cfs, DecoratedKey key, CellName name, ByteBuffer value) + private void assertDataPresent(ColumnFamilyStore cfs, DecoratedKey key, String name, ByteBuffer value) { - ColumnFamily cf = cfs.getColumnFamily(QueryFilter.getIdentityFilter(key, cfs.name, System.currentTimeMillis())); - assertFalse(cf.isEmpty()); - assertEquals(0, ByteBufferUtil.compareUnsigned(value, cf.getColumn(name).value())); + Row row = Util.getOnlyRowUnfiltered(Util.cmd(cfs, key).build()); + assertEquals(0, ByteBufferUtil.compareUnsigned(value, + row.getCell(cfs.metadata.getColumnDefinition(ByteBufferUtil.bytes(name))).value())); } private void assertNoDataPresent(ColumnFamilyStore cfs, DecoratedKey key) { - ColumnFamily cf = cfs.getColumnFamily(QueryFilter.getIdentityFilter(key, cfs.name, System.currentTimeMillis())); - assertNull(cf); + Util.assertEmpty(Util.cmd(cfs, key).build()); } } diff --git a/test/unit/org/apache/cassandra/service/QueryPagerTest.java b/test/unit/org/apache/cassandra/service/QueryPagerTest.java index b6a2b3cb04..7aa1f87f6e 100644 --- a/test/unit/org/apache/cassandra/service/QueryPagerTest.java +++ b/test/unit/org/apache/cassandra/service/QueryPagerTest.java @@ -18,33 +18,32 @@ */ package org.apache.cassandra.service; -import java.util.*; import java.nio.ByteBuffer; import java.nio.charset.CharacterCodingException; +import java.util.*; import org.junit.BeforeClass; import org.junit.Test; import org.junit.runner.RunWith; -import org.apache.cassandra.Util; -import org.apache.cassandra.SchemaLoader; -import org.apache.cassandra.OrderedJUnit4ClassRunner; +import org.apache.cassandra.*; import org.apache.cassandra.config.CFMetaData; import org.apache.cassandra.config.KSMetaData; import org.apache.cassandra.db.*; -import org.apache.cassandra.db.composites.*; +import org.apache.cassandra.db.rows.Row; +import org.apache.cassandra.db.rows.RowIterator; import org.apache.cassandra.db.filter.*; -import org.apache.cassandra.db.marshal.CompositeType; -import org.apache.cassandra.dht.*; +import org.apache.cassandra.db.partitions.FilteredPartition; +import org.apache.cassandra.db.partitions.PartitionIterator; import org.apache.cassandra.exceptions.ConfigurationException; import org.apache.cassandra.locator.SimpleStrategy; -import org.apache.cassandra.service.pager.*; +import org.apache.cassandra.service.pager.QueryPager; import org.apache.cassandra.utils.ByteBufferUtil; +import org.apache.cassandra.utils.FBUtilities; -import static org.junit.Assert.*; import static org.apache.cassandra.cql3.QueryProcessor.executeInternal; -import static org.apache.cassandra.Util.range; import static org.apache.cassandra.utils.ByteBufferUtil.bytes; +import static org.junit.Assert.*; @RunWith(OrderedJUnit4ClassRunner.class) public class QueryPagerTest @@ -66,18 +65,13 @@ public class QueryPagerTest SimpleStrategy.class, KSMetaData.optsWithRF(1), CFMetaData.compile("CREATE TABLE " + CF_CQL + " (" - + "k text," - + "c text," - + "v text," - + "PRIMARY KEY (k, c))", KEYSPACE_CQL)); + + "k text," + + "c text," + + "v text," + + "PRIMARY KEY (k, c))", KEYSPACE_CQL)); addData(); } - private static String string(CellName name) - { - return string(name.toByteBuffer()); - } - private static String string(ByteBuffer bb) { try @@ -97,21 +91,19 @@ public class QueryPagerTest int nbKeys = 10; int nbCols = 10; - /* - * Creates the following data: - * k1: c1 ... cn - * ... - * ki: c1 ... cn - */ + // * + // * Creates the following data: + // * k1: c1 ... cn + // * ... + // * ki: c1 ... cn + // * for (int i = 0; i < nbKeys; i++) { - Mutation rm = new Mutation(KEYSPACE1, bytes("k" + i)); - ColumnFamily cf = rm.addOrGet(CF_STANDARD); - for (int j = 0; j < nbCols; j++) - cf.addColumn(Util.column("c" + j, "", 0)); - - rm.applyUnsafe(); + { + RowUpdateBuilder builder = new RowUpdateBuilder(cfs().metadata, FBUtilities.timestampMicros(), "k" + i); + builder.clustering("c" + j).add("val", "").build().applyUnsafe(); + } } } @@ -120,60 +112,83 @@ public class QueryPagerTest return Keyspace.open(KEYSPACE1).getColumnFamilyStore(CF_STANDARD); } - private static String toString(List rows) + private static List query(QueryPager pager, int expectedSize) { - StringBuilder sb = new StringBuilder(); - for (Row row : rows) - sb.append(string(row.key.getKey())).append(":").append(toString(row.cf)).append("\n"); - return sb.toString(); + return query(pager, expectedSize, expectedSize); } - private static String toString(ColumnFamily cf) + private static List query(QueryPager pager, int toQuery, int expectedSize) { - if (cf == null) - return ""; - StringBuilder sb = new StringBuilder(); - for (Cell c : cf) - sb.append(" ").append(string(c.name())); - return sb.toString(); + List partitionList = new ArrayList<>(); + int rows = 0; + try (ReadOrderGroup orderGroup = pager.startOrderGroup(); PartitionIterator iterator = pager.fetchPageInternal(toQuery, orderGroup)) + { + while (iterator.hasNext()) + { + try (RowIterator rowIter = iterator.next()) + { + FilteredPartition partition = FilteredPartition.create(rowIter); + sb.append(partition); + partitionList.add(partition); + rows += partition.rowCount(); + } + } + } + assertEquals(sb.toString(), expectedSize, rows); + return partitionList; } private static ReadCommand namesQuery(String key, String... names) { - SortedSet s = new TreeSet(cfs().metadata.comparator); + AbstractReadCommandBuilder builder = Util.cmd(cfs(), key); for (String name : names) - s.add(CellNames.simpleDense(bytes(name))); - return new SliceByNamesReadCommand(KEYSPACE1, bytes(key), CF_STANDARD, System.currentTimeMillis(), new NamesQueryFilter(s, true)); + builder.includeRow(name); + return builder.withPagingLimit(100).build(); } - private static ReadCommand sliceQuery(String key, String start, String end, int count) + private static SinglePartitionSliceCommand sliceQuery(String key, String start, String end, int count) { return sliceQuery(key, start, end, false, count); } - private static ReadCommand sliceQuery(String key, String start, String end, boolean reversed, int count) + private static SinglePartitionSliceCommand sliceQuery(String key, String start, String end, boolean reversed, int count) { - SliceQueryFilter filter = new SliceQueryFilter(CellNames.simpleDense(bytes(start)), CellNames.simpleDense(bytes(end)), reversed, count); - // Note: for MultiQueryTest, we need the same timestamp/expireBefore for all queries, so we just use 0 as it doesn't matter here. - return new SliceFromReadCommand(KEYSPACE1, bytes(key), CF_STANDARD, 0, filter); + ClusteringComparator cmp = cfs().getComparator(); + CFMetaData metadata = cfs().metadata; + + Slice slice = Slice.make(cmp.make(start), cmp.make(end)); + ClusteringIndexSliceFilter filter = new ClusteringIndexSliceFilter(Slices.with(cmp, slice), reversed); + + SinglePartitionSliceCommand command = new SinglePartitionSliceCommand(cfs().metadata, FBUtilities.nowInSeconds(), ColumnFilter.all(metadata), RowFilter.NONE, DataLimits.NONE, Util.dk(key), filter); + + return command; } - private static RangeSliceCommand rangeNamesQuery(AbstractBounds range, int count, String... names) + private static ReadCommand rangeNamesQuery(String keyStart, String keyEnd, int count, String... names) { - SortedSet s = new TreeSet(cfs().metadata.comparator); + AbstractReadCommandBuilder builder = Util.cmd(cfs()) + .fromKeyExcl(keyStart) + .toKeyIncl(keyEnd) + .withPagingLimit(count); for (String name : names) - s.add(CellNames.simpleDense(bytes(name))); - return new RangeSliceCommand(KEYSPACE1, CF_STANDARD, System.currentTimeMillis(), new NamesQueryFilter(s, true), range, count); + builder.includeRow(name); + + return builder.build(); } - private static RangeSliceCommand rangeSliceQuery(AbstractBounds range, int count, String start, String end) + private static ReadCommand rangeSliceQuery(String keyStart, String keyEnd, int count, String start, String end) { - SliceQueryFilter filter = new SliceQueryFilter(CellNames.simpleDense(bytes(start)), CellNames.simpleDense(bytes(end)), false, Integer.MAX_VALUE); - return new RangeSliceCommand(KEYSPACE1, CF_STANDARD, System.currentTimeMillis(), filter, range, count); + return Util.cmd(cfs()) + .fromKeyExcl(keyStart) + .toKeyIncl(keyEnd) + .fromIncl(start) + .toIncl(end) + .withPagingLimit(count) + .build(); } - private static void assertRow(Row r, String key, String... names) + private static void assertRow(FilteredPartition r, String key, String... names) { ByteBuffer[] bbs = new ByteBuffer[names.length]; for (int i = 0; i < names.length; i++) @@ -181,31 +196,26 @@ public class QueryPagerTest assertRow(r, key, bbs); } - private static void assertRow(Row r, String key, ByteBuffer... names) + private static void assertRow(FilteredPartition partition, String key, ByteBuffer... names) { - assertEquals(key, string(r.key.getKey())); - assertNotNull(r.cf); + assertEquals(key, string(partition.partitionKey().getKey())); + assertFalse(partition.isEmpty()); int i = 0; - for (Cell c : r.cf) + for (Row row : Util.once(partition.iterator())) { - // Ignore deleted cells if we have them - if (!c.isLive()) - continue; - ByteBuffer expected = names[i++]; - assertEquals("column " + i + " doesn't match: " + toString(r.cf), expected, c.name().toByteBuffer()); + assertEquals("column " + i + " doesn't match "+string(expected)+" vs "+string(row.clustering().get(0)), expected, row.clustering().get(0)); } } @Test public void namesQueryTest() throws Exception { - QueryPager pager = QueryPagers.localPager(namesQuery("k0", "c1", "c5", "c7", "c8")); + QueryPager pager = namesQuery("k0", "c1", "c5", "c7", "c8").getPager(null); assertFalse(pager.isExhausted()); - List page = pager.fetchPage(3); - assertEquals(toString(page), 1, page.size()); - assertRow(page.get(0), "k0", "c1", "c5", "c7", "c8"); + List partition = query(pager, 5, 4); + assertRow(partition.get(0), "k0", "c1", "c5", "c7", "c8"); assertTrue(pager.isExhausted()); } @@ -213,24 +223,19 @@ public class QueryPagerTest @Test public void sliceQueryTest() throws Exception { - QueryPager pager = QueryPagers.localPager(sliceQuery("k0", "c1", "c8", 10)); - - List page; + QueryPager pager = sliceQuery("k0", "c1", "c8", 10).getPager(null); assertFalse(pager.isExhausted()); - page = pager.fetchPage(3); - assertEquals(toString(page), 1, page.size()); - assertRow(page.get(0), "k0", "c1", "c2", "c3"); + List partition = query(pager, 3); + assertRow(partition.get(0), "k0", "c1", "c2", "c3"); assertFalse(pager.isExhausted()); - page = pager.fetchPage(3); - assertEquals(toString(page), 1, page.size()); - assertRow(page.get(0), "k0", "c4", "c5", "c6"); + partition = query(pager, 3); + assertRow(partition.get(0), "k0", "c4", "c5", "c6"); assertFalse(pager.isExhausted()); - page = pager.fetchPage(3); - assertEquals(toString(page), 1, page.size()); - assertRow(page.get(0), "k0", "c7", "c8"); + partition = query(pager, 3, 2); + assertRow(partition.get(0), "k0", "c7", "c8"); assertTrue(pager.isExhausted()); } @@ -238,24 +243,19 @@ public class QueryPagerTest @Test public void reversedSliceQueryTest() throws Exception { - QueryPager pager = QueryPagers.localPager(sliceQuery("k0", "c8", "c1", true, 10)); - - List page; + QueryPager pager = sliceQuery("k0", "c1", "c8", true, 10).getPager(null); assertFalse(pager.isExhausted()); - page = pager.fetchPage(3); - assertEquals(toString(page), 1, page.size()); - assertRow(page.get(0), "k0", "c6", "c7", "c8"); + List partition = query(pager, 3); + assertRow(partition.get(0), "k0", "c6", "c7", "c8"); assertFalse(pager.isExhausted()); - page = pager.fetchPage(3); - assertEquals(toString(page), 1, page.size()); - assertRow(page.get(0), "k0", "c3", "c4", "c5"); + partition = query(pager, 3); + assertRow(partition.get(0), "k0", "c3", "c4", "c5"); assertFalse(pager.isExhausted()); - page = pager.fetchPage(3); - assertEquals(toString(page), 1, page.size()); - assertRow(page.get(0), "k0", "c1", "c2"); + partition = query(pager, 3, 2); + assertRow(partition.get(0), "k0", "c1", "c2"); assertTrue(pager.isExhausted()); } @@ -263,28 +263,24 @@ public class QueryPagerTest @Test public void multiQueryTest() throws Exception { - QueryPager pager = QueryPagers.localPager(new Pageable.ReadCommands(new ArrayList() {{ + QueryPager pager = new SinglePartitionReadCommand.Group(new ArrayList>() + {{ add(sliceQuery("k1", "c2", "c6", 10)); add(sliceQuery("k4", "c3", "c5", 10)); - }}, 10)); - - List page; + }}, DataLimits.NONE).getPager(null); assertFalse(pager.isExhausted()); - page = pager.fetchPage(3); - assertEquals(toString(page), 1, page.size()); - assertRow(page.get(0), "k1", "c2", "c3", "c4"); + List partition = query(pager, 3); + assertRow(partition.get(0), "k1", "c2", "c3", "c4"); assertFalse(pager.isExhausted()); - page = pager.fetchPage(4); - assertEquals(toString(page), 2, page.size()); - assertRow(page.get(0), "k1", "c5", "c6"); - assertRow(page.get(1), "k4", "c3", "c4"); + partition = query(pager , 4); + assertRow(partition.get(0), "k1", "c5", "c6"); + assertRow(partition.get(1), "k4", "c3", "c4"); assertFalse(pager.isExhausted()); - page = pager.fetchPage(3); - assertEquals(toString(page), 1, page.size()); - assertRow(page.get(0), "k4", "c5"); + partition = query(pager, 3, 1); + assertRow(partition.get(0), "k4", "c5"); assertTrue(pager.isExhausted()); } @@ -292,21 +288,17 @@ public class QueryPagerTest @Test public void rangeNamesQueryTest() throws Exception { - QueryPager pager = QueryPagers.localPager(rangeNamesQuery(range("k0", "k5"), 100, "c1", "c4", "c8")); - - List page; + QueryPager pager = rangeNamesQuery("k0", "k5", 100, "c1", "c4", "c8").getPager(null); assertFalse(pager.isExhausted()); - page = pager.fetchPage(3); - assertEquals(toString(page), 3, page.size()); + List partitions = query(pager, 3 * 3); for (int i = 1; i <= 3; i++) - assertRow(page.get(i-1), "k" + i, "c1", "c4", "c8"); + assertRow(partitions.get(i-1), "k" + i, "c1", "c4", "c8"); assertFalse(pager.isExhausted()); - page = pager.fetchPage(3); - assertEquals(toString(page), 2, page.size()); + partitions = query(pager, 3 * 3, 2 * 3); for (int i = 4; i <= 5; i++) - assertRow(page.get(i-4), "k" + i, "c1", "c4", "c8"); + assertRow(partitions.get(i-4), "k" + i, "c1", "c4", "c8"); assertTrue(pager.isExhausted()); } @@ -314,46 +306,39 @@ public class QueryPagerTest @Test public void rangeSliceQueryTest() throws Exception { - QueryPager pager = QueryPagers.localPager(rangeSliceQuery(range("k1", "k5"), 100, "c1", "c7")); - - List page; + QueryPager pager = rangeSliceQuery("k1", "k5", 100, "c1", "c7").getPager(null); assertFalse(pager.isExhausted()); - page = pager.fetchPage(5); - assertEquals(toString(page), 1, page.size()); - assertRow(page.get(0), "k2", "c1", "c2", "c3", "c4", "c5"); + List partitions = query(pager, 5); + assertRow(partitions.get(0), "k2", "c1", "c2", "c3", "c4", "c5"); assertFalse(pager.isExhausted()); - page = pager.fetchPage(4); - assertEquals(toString(page), 2, page.size()); - assertRow(page.get(0), "k2", "c6", "c7"); - assertRow(page.get(1), "k3", "c1", "c2"); + partitions = query(pager, 4); + assertRow(partitions.get(0), "k2", "c6", "c7"); + assertRow(partitions.get(1), "k3", "c1", "c2"); assertFalse(pager.isExhausted()); - page = pager.fetchPage(6); - assertEquals(toString(page), 2, page.size()); - assertRow(page.get(0), "k3", "c3", "c4", "c5", "c6", "c7"); - assertRow(page.get(1), "k4", "c1"); + partitions = query(pager, 6); + assertRow(partitions.get(0), "k3", "c3", "c4", "c5", "c6", "c7"); + assertRow(partitions.get(1), "k4", "c1"); assertFalse(pager.isExhausted()); - page = pager.fetchPage(5); - assertEquals(toString(page), 1, page.size()); - assertRow(page.get(0), "k4", "c2", "c3", "c4", "c5", "c6"); + partitions = query(pager, 5); + assertRow(partitions.get(0), "k4", "c2", "c3", "c4", "c5", "c6"); assertFalse(pager.isExhausted()); - page = pager.fetchPage(5); - assertEquals(toString(page), 2, page.size()); - assertRow(page.get(0), "k4", "c7"); - assertRow(page.get(1), "k5", "c1", "c2", "c3", "c4"); + partitions = query(pager, 5); + assertRow(partitions.get(0), "k4", "c7"); + assertRow(partitions.get(1), "k5", "c1", "c2", "c3", "c4"); assertFalse(pager.isExhausted()); - page = pager.fetchPage(5); - assertEquals(toString(page), 1, page.size()); - assertRow(page.get(0), "k5", "c5", "c6", "c7"); + partitions = query(pager, 5, 3); + assertRow(partitions.get(0), "k5", "c5", "c6", "c7"); assertTrue(pager.isExhausted()); } + @Test public void SliceQueryWithTombstoneTest() throws Exception { @@ -361,21 +346,20 @@ public class QueryPagerTest String keyspace = "cql_keyspace"; String table = "table2"; ColumnFamilyStore cfs = Keyspace.open(keyspace).getColumnFamilyStore(table); - CompositeType ct = (CompositeType)cfs.metadata.comparator.asAbstractType(); // Insert rows but with a tombstone as last cell for (int i = 0; i < 5; i++) executeInternal(String.format("INSERT INTO %s.%s (k, c, v) VALUES ('k%d', 'c%d', null)", keyspace, table, 0, i)); - SliceQueryFilter filter = new SliceQueryFilter(ColumnSlice.ALL_COLUMNS_ARRAY, false, 100); - QueryPager pager = QueryPagers.localPager(new SliceFromReadCommand(keyspace, bytes("k0"), table, 0, filter)); + ReadCommand command = SinglePartitionSliceCommand.create(cfs.metadata, FBUtilities.nowInSeconds(), Util.dk("k0"), Slice.ALL); + + QueryPager pager = command.getPager(null); for (int i = 0; i < 5; i++) { - List page = pager.fetchPage(1); - assertEquals(toString(page), 1, page.size()); + List partitions = query(pager, 1); // The only live cell we should have each time is the row marker - assertRow(page.get(0), "k0", ct.decompose("c" + i, "")); + assertRow(partitions.get(0), "k0", "c" + i); } } } diff --git a/test/unit/org/apache/cassandra/service/RowResolverTest.java b/test/unit/org/apache/cassandra/service/RowResolverTest.java deleted file mode 100644 index 825944c1c5..0000000000 --- a/test/unit/org/apache/cassandra/service/RowResolverTest.java +++ /dev/null @@ -1,160 +0,0 @@ -package org.apache.cassandra.service; -/* - * - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - * - */ - - -import java.util.Arrays; - -import org.junit.BeforeClass; -import org.junit.Test; - -import org.apache.cassandra.SchemaLoader; -import org.apache.cassandra.config.KSMetaData; -import org.apache.cassandra.db.ArrayBackedSortedColumns; -import org.apache.cassandra.db.ColumnFamily; -import org.apache.cassandra.db.DeletionInfo; -import org.apache.cassandra.exceptions.ConfigurationException; -import org.apache.cassandra.locator.SimpleStrategy; - -import static org.junit.Assert.*; -import static org.apache.cassandra.Util.column; -import static org.apache.cassandra.db.KeyspaceTest.*; - -public class RowResolverTest -{ - public static final String KEYSPACE1 = "Keyspace1"; - public static final String CF_STANDARD = "Standard1"; - - @BeforeClass - public static void defineSchema() throws ConfigurationException - { - SchemaLoader.prepareServer(); - SchemaLoader.createKeyspace(KEYSPACE1, - SimpleStrategy.class, - KSMetaData.optsWithRF(1), - SchemaLoader.standardCFMD(KEYSPACE1, CF_STANDARD)); - } - - @Test - public void testResolveSupersetNewer() - { - ColumnFamily cf1 = ArrayBackedSortedColumns.factory.create(KEYSPACE1, "Standard1"); - cf1.addColumn(column("c1", "v1", 0)); - - ColumnFamily cf2 = ArrayBackedSortedColumns.factory.create(KEYSPACE1, "Standard1"); - cf2.addColumn(column("c1", "v2", 1)); - - ColumnFamily resolved = RowDataResolver.resolveSuperset(Arrays.asList(cf1, cf2), System.currentTimeMillis()); - assertColumns(resolved, "c1"); - assertColumns(ColumnFamily.diff(cf1, resolved), "c1"); - assertNull(ColumnFamily.diff(cf2, resolved)); - } - - @Test - public void testResolveSupersetDisjoint() - { - ColumnFamily cf1 = ArrayBackedSortedColumns.factory.create(KEYSPACE1, "Standard1"); - cf1.addColumn(column("c1", "v1", 0)); - - ColumnFamily cf2 = ArrayBackedSortedColumns.factory.create(KEYSPACE1, "Standard1"); - cf2.addColumn(column("c2", "v2", 1)); - - ColumnFamily resolved = RowDataResolver.resolveSuperset(Arrays.asList(cf1, cf2), System.currentTimeMillis()); - assertColumns(resolved, "c1", "c2"); - assertColumns(ColumnFamily.diff(cf1, resolved), "c2"); - assertColumns(ColumnFamily.diff(cf2, resolved), "c1"); - } - - @Test - public void testResolveSupersetNullOne() - { - ColumnFamily cf2 = ArrayBackedSortedColumns.factory.create(KEYSPACE1, "Standard1"); - cf2.addColumn(column("c2", "v2", 1)); - - ColumnFamily resolved = RowDataResolver.resolveSuperset(Arrays.asList(null, cf2), System.currentTimeMillis()); - assertColumns(resolved, "c2"); - assertColumns(ColumnFamily.diff(null, resolved), "c2"); - assertNull(ColumnFamily.diff(cf2, resolved)); - } - - @Test - public void testResolveSupersetNullTwo() - { - ColumnFamily cf1 = ArrayBackedSortedColumns.factory.create(KEYSPACE1, "Standard1"); - cf1.addColumn(column("c1", "v1", 0)); - - ColumnFamily resolved = RowDataResolver.resolveSuperset(Arrays.asList(cf1, null), System.currentTimeMillis()); - assertColumns(resolved, "c1"); - assertNull(ColumnFamily.diff(cf1, resolved)); - assertColumns(ColumnFamily.diff(null, resolved), "c1"); - } - - @Test - public void testResolveSupersetNullBoth() - { - assertNull(RowDataResolver.resolveSuperset(Arrays.asList(null, null), System.currentTimeMillis())); - } - - @Test - public void testResolveDeleted() - { - // one CF with columns timestamped before a delete in another cf - ColumnFamily cf1 = ArrayBackedSortedColumns.factory.create(KEYSPACE1, "Standard1"); - cf1.addColumn(column("one", "A", 0)); - - ColumnFamily cf2 = ArrayBackedSortedColumns.factory.create(KEYSPACE1, "Standard1"); - cf2.delete(new DeletionInfo(1L, (int) (System.currentTimeMillis() / 1000))); - - ColumnFamily resolved = RowDataResolver.resolveSuperset(Arrays.asList(cf1, cf2), System.currentTimeMillis()); - // no columns in the cf - assertColumns(resolved); - assertTrue(resolved.isMarkedForDelete()); - assertEquals(1, resolved.deletionInfo().getTopLevelDeletion().markedForDeleteAt); - } - - @Test - public void testResolveMultipleDeleted() - { - // deletes and columns with interleaved timestamp, with out of order return sequence - - ColumnFamily cf1 = ArrayBackedSortedColumns.factory.create(KEYSPACE1, "Standard1"); - cf1.delete(new DeletionInfo(0L, (int) (System.currentTimeMillis() / 1000))); - - // these columns created after the previous deletion - ColumnFamily cf2 = ArrayBackedSortedColumns.factory.create(KEYSPACE1, "Standard1"); - cf2.addColumn(column("one", "A", 1)); - cf2.addColumn(column("two", "A", 1)); - - //this column created after the next delete - ColumnFamily cf3 = ArrayBackedSortedColumns.factory.create(KEYSPACE1, "Standard1"); - cf3.addColumn(column("two", "B", 3)); - - ColumnFamily cf4 = ArrayBackedSortedColumns.factory.create(KEYSPACE1, "Standard1"); - cf4.delete(new DeletionInfo(2L, (int) (System.currentTimeMillis() / 1000))); - - ColumnFamily resolved = RowDataResolver.resolveSuperset(Arrays.asList(cf1, cf2, cf3, cf4), System.currentTimeMillis()); - // will have deleted marker and one column - assertColumns(resolved, "two"); - assertColumn(resolved, "two", "B", 3); - assertTrue(resolved.isMarkedForDelete()); - assertEquals(2, resolved.deletionInfo().getTopLevelDeletion().markedForDeleteAt); - } -} diff --git a/test/unit/org/apache/cassandra/service/SerializationsTest.java b/test/unit/org/apache/cassandra/service/SerializationsTest.java index 5d2b74d121..9c8e0fbafc 100644 --- a/test/unit/org/apache/cassandra/service/SerializationsTest.java +++ b/test/unit/org/apache/cassandra/service/SerializationsTest.java @@ -25,6 +25,7 @@ import java.util.Collections; import java.util.UUID; import org.junit.Test; + import org.apache.cassandra.AbstractSerializationsTester; import org.apache.cassandra.dht.IPartitioner; import org.apache.cassandra.dht.RandomPartitioner; diff --git a/test/unit/org/apache/cassandra/service/StorageProxyTest.java b/test/unit/org/apache/cassandra/service/StorageProxyTest.java index c8afac0e73..c996d5cf7b 100644 --- a/test/unit/org/apache/cassandra/service/StorageProxyTest.java +++ b/test/unit/org/apache/cassandra/service/StorageProxyTest.java @@ -23,49 +23,44 @@ import java.util.List; import org.junit.BeforeClass; import org.junit.Test; -import static org.junit.Assert.assertEquals; -import static org.apache.cassandra.Util.token; -import static org.apache.cassandra.Util.rp; - -import org.apache.cassandra.db.RowPosition; -import org.apache.cassandra.dht.AbstractBounds; -import org.apache.cassandra.dht.Bounds; -import org.apache.cassandra.dht.Range; -import org.apache.cassandra.dht.Token; -import org.apache.cassandra.dht.ExcludingBounds; -import org.apache.cassandra.dht.IncludingExcludingBounds; +import org.apache.cassandra.db.PartitionPosition; +import org.apache.cassandra.dht.*; import org.apache.cassandra.locator.TokenMetadata; import org.apache.cassandra.utils.ByteBufferUtil; +import static org.apache.cassandra.Util.rp; +import static org.apache.cassandra.Util.token; +import static org.junit.Assert.assertEquals; + public class StorageProxyTest { - private static Range range(RowPosition left, RowPosition right) + private static Range range(PartitionPosition left, PartitionPosition right) { - return new Range(left, right); + return new Range(left, right); } - private static Bounds bounds(RowPosition left, RowPosition right) + private static Bounds bounds(PartitionPosition left, PartitionPosition right) { - return new Bounds(left, right); + return new Bounds(left, right); } - private static ExcludingBounds exBounds(RowPosition left, RowPosition right) + private static ExcludingBounds exBounds(PartitionPosition left, PartitionPosition right) { - return new ExcludingBounds(left, right); + return new ExcludingBounds(left, right); } - private static IncludingExcludingBounds incExBounds(RowPosition left, RowPosition right) + private static IncludingExcludingBounds incExBounds(PartitionPosition left, PartitionPosition right) { - return new IncludingExcludingBounds(left, right); + return new IncludingExcludingBounds(left, right); } - private static RowPosition startOf(String key) + private static PartitionPosition startOf(String key) { return StorageService.getPartitioner().getToken(ByteBufferUtil.bytes(key)).minKeyBound(); } - private static RowPosition endOf(String key) + private static PartitionPosition endOf(String key) { return StorageService.getPartitioner().getToken(ByteBufferUtil.bytes(key)).maxKeyBound(); } @@ -99,10 +94,10 @@ public class StorageProxyTest } // test getRestrictedRanges for keys - private void testGRRKeys(AbstractBounds queryRange, AbstractBounds... expected) + private void testGRRKeys(AbstractBounds queryRange, AbstractBounds... expected) { // Testing for keys - List> restrictedKeys = StorageProxy.getRestrictedRanges(queryRange); + List> restrictedKeys = StorageProxy.getRestrictedRanges(queryRange); assertEquals(restrictedKeys.toString(), expected.length, restrictedKeys.size()); for (int i = 0; i < expected.length; i++) assertEquals("Mismatch for index " + i + ": " + restrictedKeys, expected[i], restrictedKeys.get(i)); diff --git a/test/unit/org/apache/cassandra/service/StorageServiceServerTest.java b/test/unit/org/apache/cassandra/service/StorageServiceServerTest.java index a20fa6c2c9..24d57e7def 100644 --- a/test/unit/org/apache/cassandra/service/StorageServiceServerTest.java +++ b/test/unit/org/apache/cassandra/service/StorageServiceServerTest.java @@ -26,7 +26,6 @@ import java.util.*; import com.google.common.collect.HashMultimap; import com.google.common.collect.Multimap; - import org.junit.BeforeClass; import org.junit.Test; import org.junit.runner.RunWith; @@ -37,17 +36,17 @@ import org.apache.cassandra.config.DatabaseDescriptor; import org.apache.cassandra.config.KSMetaData; import org.apache.cassandra.config.Schema; import org.apache.cassandra.db.Keyspace; -import org.apache.cassandra.schema.LegacySchemaTables; import org.apache.cassandra.db.SystemKeyspace; -import org.apache.cassandra.dht.Range; -import org.apache.cassandra.dht.Token; -import org.apache.cassandra.dht.OrderPreservingPartitioner.StringToken; import org.apache.cassandra.dht.Murmur3Partitioner; import org.apache.cassandra.dht.Murmur3Partitioner.LongToken; +import org.apache.cassandra.dht.OrderPreservingPartitioner.StringToken; +import org.apache.cassandra.dht.Range; +import org.apache.cassandra.dht.Token; import org.apache.cassandra.exceptions.ConfigurationException; import org.apache.cassandra.locator.IEndpointSnitch; import org.apache.cassandra.locator.PropertyFileSnitch; import org.apache.cassandra.locator.TokenMetadata; +import org.apache.cassandra.schema.LegacySchemaTables; import static org.junit.Assert.assertEquals; import static org.junit.Assert.assertTrue; diff --git a/test/unit/org/apache/cassandra/service/pager/AbstractQueryPagerTest.java b/test/unit/org/apache/cassandra/service/pager/AbstractQueryPagerTest.java deleted file mode 100644 index 00718b4ce4..0000000000 --- a/test/unit/org/apache/cassandra/service/pager/AbstractQueryPagerTest.java +++ /dev/null @@ -1,187 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.apache.cassandra.service.pager; - -import java.nio.ByteBuffer; -import java.util.*; - -import org.junit.Test; -import static org.junit.Assert.*; - -import org.apache.cassandra.Util; -import org.apache.cassandra.config.*; -import org.apache.cassandra.db.*; -import org.apache.cassandra.db.composites.CellNames; -import org.apache.cassandra.db.filter.ColumnCounter; -import org.apache.cassandra.db.marshal.Int32Type; -import org.apache.cassandra.utils.ByteBufferUtil; - -public class AbstractQueryPagerTest -{ - @Test - public void discardFirstTest() - { - TestPager pager = new TestPager(); - List rows = Arrays.asList(createRow("r1", 1), - createRow("r2", 3), - createRow("r3", 2)); - - assertEquals(3, rows.size()); - assertRow(rows.get(0), "r1", 0); - assertRow(rows.get(1), "r2", 0, 1, 2); - assertRow(rows.get(2), "r3", 0, 1); - - rows = pager.discardFirst(rows, 1); - - assertEquals(2, rows.size()); - assertRow(rows.get(0), "r2", 0, 1, 2); - assertRow(rows.get(1), "r3", 0, 1); - - rows = pager.discardFirst(rows, 1); - - assertEquals(2, rows.size()); - assertRow(rows.get(0), "r2", 1, 2); - assertRow(rows.get(1), "r3", 0, 1); - - rows = pager.discardFirst(rows, 3); - - assertEquals(1, rows.size()); - assertRow(rows.get(0), "r3", 1); - - rows = pager.discardFirst(rows, 1); - - assertTrue(rows.isEmpty()); - } - - @Test - public void discardLastTest() - { - TestPager pager = new TestPager(); - List rows = Arrays.asList(createRow("r1", 2), - createRow("r2", 3), - createRow("r3", 1)); - - assertEquals(3, rows.size()); - assertRow(rows.get(0), "r1", 0, 1); - assertRow(rows.get(1), "r2", 0, 1, 2); - assertRow(rows.get(2), "r3", 0); - - rows = pager.discardLast(rows, 1); - - assertEquals(2, rows.size()); - assertRow(rows.get(0), "r1", 0, 1); - assertRow(rows.get(1), "r2", 0, 1, 2); - - rows = pager.discardLast(rows, 1); - - assertEquals(2, rows.size()); - assertRow(rows.get(0), "r1", 0, 1); - assertRow(rows.get(1), "r2", 0, 1); - - rows = pager.discardLast(rows, 3); - - assertEquals(1, rows.size()); - assertRow(rows.get(0), "r1", 0); - - rows = pager.discardLast(rows, 1); - - assertTrue(rows.isEmpty()); - } - - private void assertRow(Row row, String name, int... values) - { - assertEquals(row.key.getKey(), ByteBufferUtil.bytes(name)); - assertEquals(values.length, row.cf.getColumnCount()); - - int i = 0; - for (Cell c : row.cf) - assertEquals(values[i++], i(c.name().toByteBuffer())); - } - - private Row createRow(String name, int nbCol) - { - return new Row(Util.dk(name), createCF(nbCol)); - } - - private ColumnFamily createCF(int nbCol) - { - ColumnFamily cf = ArrayBackedSortedColumns.factory.create(createMetadata()); - for (int i = 0; i < nbCol; i++) - cf.addColumn(CellNames.simpleDense(bb(i)), bb(i), 0); - return cf; - } - - private static CFMetaData createMetadata() - { - CFMetaData cfm = new CFMetaData("ks", "cf", ColumnFamilyType.Standard, CellNames.fromAbstractType(Int32Type.instance, false)); - cfm.rebuild(); - return cfm; - } - - private static ByteBuffer bb(int i) - { - return ByteBufferUtil.bytes(i); - } - - private static int i(ByteBuffer bb) - { - return ByteBufferUtil.toInt(bb); - } - - private static class TestPager extends AbstractQueryPager - { - public TestPager() - { - // We use this to test more thorougly DiscardFirst and DiscardLast (more generic pager behavior is tested in - // QueryPagerTest). The only thing those method use is the result of the columnCounter() method. So to keep - // it simple, we fake all actual parameters in the ctor below but just override the columnCounter() method. - super(null, 0, false, createMetadata(), null, 0); - } - - @Override - public ColumnCounter columnCounter() - { - return new ColumnCounter(0); - } - - public PagingState state() - { - return null; - } - - protected List queryNextPage(int pageSize, ConsistencyLevel consistency, boolean localQuery) - { - return null; - } - - protected boolean containsPreviousLast(Row first) - { - return false; - } - - protected boolean recordLast(Row last) - { - return false; - } - - protected boolean isReversed() - { - return false; - } - } -} diff --git a/test/unit/org/apache/cassandra/streaming/StreamTransferTaskTest.java b/test/unit/org/apache/cassandra/streaming/StreamTransferTaskTest.java index c3c16b8562..06c9fc0626 100644 --- a/test/unit/org/apache/cassandra/streaming/StreamTransferTaskTest.java +++ b/test/unit/org/apache/cassandra/streaming/StreamTransferTaskTest.java @@ -24,7 +24,6 @@ import java.util.concurrent.CancellationException; import java.util.concurrent.Future; import java.util.concurrent.TimeUnit; -import org.apache.cassandra.io.sstable.format.SSTableReader; import org.junit.BeforeClass; import org.junit.Test; @@ -36,6 +35,7 @@ import org.apache.cassandra.db.Keyspace; import org.apache.cassandra.dht.Range; import org.apache.cassandra.dht.Token; import org.apache.cassandra.exceptions.ConfigurationException; +import org.apache.cassandra.io.sstable.format.SSTableReader; import org.apache.cassandra.locator.SimpleStrategy; import org.apache.cassandra.utils.FBUtilities; @@ -52,9 +52,9 @@ public class StreamTransferTaskTest { SchemaLoader.prepareServer(); SchemaLoader.createKeyspace(KEYSPACE1, - SimpleStrategy.class, - KSMetaData.optsWithRF(1), - SchemaLoader.standardCFMD(KEYSPACE1, CF_STANDARD)); + SimpleStrategy.class, + KSMetaData.optsWithRF(1), + SchemaLoader.standardCFMD(KEYSPACE1, CF_STANDARD)); } @Test diff --git a/test/unit/org/apache/cassandra/streaming/StreamingTransferTest.java b/test/unit/org/apache/cassandra/streaming/StreamingTransferTest.java index 3c799e245c..3fe8d7aac4 100644 --- a/test/unit/org/apache/cassandra/streaming/StreamingTransferTest.java +++ b/test/unit/org/apache/cassandra/streaming/StreamingTransferTest.java @@ -19,50 +19,43 @@ package org.apache.cassandra.streaming; import java.net.InetAddress; import java.nio.ByteBuffer; -import java.sql.Date; import java.util.*; import java.util.concurrent.TimeUnit; import com.google.common.collect.Iterables; import com.google.common.util.concurrent.FutureCallback; import com.google.common.util.concurrent.Futures; -import org.apache.cassandra.io.sstable.format.SSTableReader; import org.junit.BeforeClass; import org.junit.Test; import org.junit.runner.RunWith; import org.slf4j.Logger; import org.slf4j.LoggerFactory; +import junit.framework.Assert; import org.apache.cassandra.OrderedJUnit4ClassRunner; import org.apache.cassandra.SchemaLoader; import org.apache.cassandra.Util; import org.apache.cassandra.config.CFMetaData; import org.apache.cassandra.config.DatabaseDescriptor; import org.apache.cassandra.config.KSMetaData; -import org.apache.cassandra.cql3.Operator; +import org.apache.cassandra.cql3.QueryProcessor; +import org.apache.cassandra.cql3.UntypedResultSet; import org.apache.cassandra.db.*; -import org.apache.cassandra.db.columniterator.IdentityQueryFilter; -import org.apache.cassandra.db.context.CounterContext; -import org.apache.cassandra.db.filter.IDiskAtomFilter; -import org.apache.cassandra.db.filter.QueryFilter; -import org.apache.cassandra.db.marshal.BytesType; -import org.apache.cassandra.db.marshal.CounterColumnType; -import org.apache.cassandra.db.marshal.IntegerType; +import org.apache.cassandra.db.rows.*; +import org.apache.cassandra.db.marshal.*; +import org.apache.cassandra.db.partitions.*; import org.apache.cassandra.dht.IPartitioner; import org.apache.cassandra.dht.Range; import org.apache.cassandra.dht.Token; -import org.apache.cassandra.io.sstable.SSTableUtils; +import org.apache.cassandra.io.sstable.format.SSTableReader; import org.apache.cassandra.locator.SimpleStrategy; import org.apache.cassandra.service.StorageService; import org.apache.cassandra.utils.ByteBufferUtil; -import org.apache.cassandra.utils.CounterId; import org.apache.cassandra.utils.FBUtilities; - import org.apache.cassandra.utils.concurrent.Refs; + import static org.junit.Assert.assertEquals; import static org.junit.Assert.fail; -import static org.apache.cassandra.Util.cellname; -import static org.apache.cassandra.Util.column; @RunWith(OrderedJUnit4ClassRunner.class) public class StreamingTransferTest @@ -86,21 +79,27 @@ public class StreamingTransferTest SchemaLoader.prepareServer(); StorageService.instance.initServer(); SchemaLoader.createKeyspace(KEYSPACE1, - SimpleStrategy.class, - KSMetaData.optsWithRF(1), - SchemaLoader.standardCFMD(KEYSPACE1, CF_STANDARD), - SchemaLoader.standardCFMD(KEYSPACE1, CF_COUNTER).defaultValidator(CounterColumnType.instance), - SchemaLoader.standardCFMD(KEYSPACE1, CF_STANDARDINT, IntegerType.instance), - SchemaLoader.indexCFMD(KEYSPACE1, CF_INDEX, true)); + SimpleStrategy.class, + KSMetaData.optsWithRF(1), + SchemaLoader.standardCFMD(KEYSPACE1, CF_STANDARD), + CFMetaData.Builder.create(KEYSPACE1, CF_COUNTER, false, true, true) + .addPartitionKey("key", BytesType.instance) + .build(), + CFMetaData.Builder.create(KEYSPACE1, CF_STANDARDINT) + .addPartitionKey("key", AsciiType.instance) + .addClusteringColumn("cols", Int32Type.instance) + .addRegularColumn("val", BytesType.instance) + .build(), + SchemaLoader.compositeIndexCFMD(KEYSPACE1, CF_INDEX, true)); SchemaLoader.createKeyspace(KEYSPACE2, - SimpleStrategy.class, - KSMetaData.optsWithRF(1)); + SimpleStrategy.class, + KSMetaData.optsWithRF(1)); SchemaLoader.createKeyspace(KEYSPACE_CACHEKEY, - SimpleStrategy.class, - KSMetaData.optsWithRF(1), - SchemaLoader.standardCFMD(KEYSPACE_CACHEKEY, CF_STANDARD), - SchemaLoader.standardCFMD(KEYSPACE_CACHEKEY, CF_STANDARD2), - SchemaLoader.standardCFMD(KEYSPACE_CACHEKEY, CF_STANDARD3)); + SimpleStrategy.class, + KSMetaData.optsWithRF(1), + SchemaLoader.standardCFMD(KEYSPACE_CACHEKEY, CF_STANDARD), + SchemaLoader.standardCFMD(KEYSPACE_CACHEKEY, CF_STANDARD2), + SchemaLoader.standardCFMD(KEYSPACE_CACHEKEY, CF_STANDARD3)); } /** @@ -194,15 +193,17 @@ public class StreamingTransferTest assertEquals(1, cfs.getSSTables().size()); // and that the index and filter were properly recovered - List rows = Util.getRangeSlice(cfs); - assertEquals(offs.length, rows.size()); + List partitions = Util.getAllUnfiltered(Util.cmd(cfs).build()); + assertEquals(offs.length, partitions.size()); for (int i = 0; i < offs.length; i++) { String key = "key" + offs[i]; String col = "col" + offs[i]; - assert cfs.getColumnFamily(QueryFilter.getIdentityFilter(Util.dk(key), cfs.name, System.currentTimeMillis())) != null; - assert rows.get(i).key.getKey().equals(ByteBufferUtil.bytes(key)); - assert rows.get(i).cf.getColumn(cellname(col)) != null; + + assert !Util.getAll(Util.cmd(cfs, key).build()).isEmpty(); + ArrayBackedPartition partition = partitions.get(i); + assert ByteBufferUtil.compareUnsigned(partition.partitionKey().getKey(), ByteBufferUtil.bytes(key)) == 0; + assert ByteBufferUtil.compareUnsigned(partition.iterator().next().clustering().get(0), ByteBufferUtil.bytes(col)) == 0; } // and that the max timestamp for the file was rediscovered @@ -254,19 +255,17 @@ public class StreamingTransferTest private void doTransferTable(boolean transferSSTables) throws Exception { final Keyspace keyspace = Keyspace.open(KEYSPACE1); - final ColumnFamilyStore cfs = keyspace.getColumnFamilyStore("Indexed1"); + final ColumnFamilyStore cfs = keyspace.getColumnFamilyStore(CF_INDEX); List keys = createAndTransfer(cfs, new Mutator() { public void mutate(String key, String col, long timestamp) throws Exception { long val = key.hashCode(); - ColumnFamily cf = ArrayBackedSortedColumns.factory.create(keyspace.getName(), cfs.name); - cf.addColumn(column(col, "v", timestamp)); - cf.addColumn(new BufferCell(cellname("birthdate"), ByteBufferUtil.bytes(val), timestamp)); - Mutation rm = new Mutation(KEYSPACE1, ByteBufferUtil.bytes(key), cf); - logger.debug("Applying row to transfer {}", rm); - rm.applyUnsafe(); + + RowUpdateBuilder builder = new RowUpdateBuilder(cfs.metadata, timestamp, key); + builder.clustering(col).add("birthdate", ByteBufferUtil.bytes(val)); + builder.build().applyUnsafe(); } }, transferSSTables); @@ -274,15 +273,13 @@ public class StreamingTransferTest for (String key : keys) { long val = key.hashCode(); - IndexExpression expr = new IndexExpression(ByteBufferUtil.bytes("birthdate"), - Operator.EQ, - ByteBufferUtil.bytes(val)); - List clause = Arrays.asList(expr); - IDiskAtomFilter filter = new IdentityQueryFilter(); - Range range = Util.range("", ""); - List rows = cfs.search(range, clause, filter, 100); - assertEquals(1, rows.size()); - assert rows.get(0).key.getKey().equals(ByteBufferUtil.bytes(key)); + + // test we can search: + UntypedResultSet result = QueryProcessor.executeInternal(String.format("SELECT * FROM \"%s\".\"%s\" WHERE birthdate = %d", + cfs.metadata.ksName, cfs.metadata.cfName, val)); + assertEquals(1, result.size()); + + assert result.iterator().next().getBytes("key").equals(ByteBufferUtil.bytes(key)); } } @@ -296,17 +293,37 @@ public class StreamingTransferTest String cfname = "StandardInteger1"; Keyspace keyspace = Keyspace.open(ks); ColumnFamilyStore cfs = keyspace.getColumnFamilyStore(cfname); + ClusteringComparator comparator = cfs.getComparator(); String key = "key1"; - Mutation rm = new Mutation(ks, ByteBufferUtil.bytes(key)); + + + RowUpdateBuilder updates = new RowUpdateBuilder(cfs.metadata, FBUtilities.timestampMicros(), key); + // add columns of size slightly less than column_index_size to force insert column index - rm.add(cfname, cellname(1), ByteBuffer.wrap(new byte[DatabaseDescriptor.getColumnIndexSize() - 64]), 2); - rm.add(cfname, cellname(6), ByteBuffer.wrap(new byte[DatabaseDescriptor.getColumnIndexSize()]), 2); - ColumnFamily cf = rm.addOrGet(cfname); + updates.clustering(1) + .add("val", ByteBuffer.wrap(new byte[DatabaseDescriptor.getColumnIndexSize() - 64])) + .build() + .apply(); + + updates = new RowUpdateBuilder(cfs.metadata, FBUtilities.timestampMicros(), key); + updates.clustering(6) + .add("val", ByteBuffer.wrap(new byte[DatabaseDescriptor.getColumnIndexSize()])) + .build() + .apply(); + // add RangeTombstones - cf.delete(new DeletionInfo(cellname(2), cellname(3), cf.getComparator(), 1, (int) (System.currentTimeMillis() / 1000))); - cf.delete(new DeletionInfo(cellname(5), cellname(7), cf.getComparator(), 1, (int) (System.currentTimeMillis() / 1000))); - rm.applyUnsafe(); + //updates = new RowUpdateBuilder(cfs.metadata, FBUtilities.timestampMicros() + 1 , key); + //updates.addRangeTombstone(Slice.make(comparator, comparator.make(2), comparator.make(4))) + // .build() + // .apply(); + + + updates = new RowUpdateBuilder(cfs.metadata, FBUtilities.timestampMicros() + 1, key); + updates.addRangeTombstone(Slice.make(comparator.make(5), comparator.make(7))) + .build() + .apply(); + cfs.forceBlockingFlush(); SSTableReader sstable = cfs.getSSTables().iterator().next(); @@ -316,8 +333,9 @@ public class StreamingTransferTest // confirm that a single SSTable was transferred and registered assertEquals(1, cfs.getSSTables().size()); - List rows = Util.getRangeSlice(cfs); - assertEquals(1, rows.size()); + Row r = Util.getOnlyRow(Util.cmd(cfs).build()); + Assert.assertFalse(r.isEmpty()); + Assert.assertTrue(1 == Int32Type.instance.compose(r.clustering().get(0))); } @Test @@ -332,6 +350,7 @@ public class StreamingTransferTest doTransferTable(true); } + /* @Test public void testTransferTableCounter() throws Exception { @@ -343,7 +362,7 @@ public class StreamingTransferTest List keys = createAndTransfer(cfs, new Mutator() { - /** Creates a new SSTable per key: all will be merged before streaming. */ + // Creates a new SSTable per key: all will be merged before streaming. public void mutate(String key, String col, long timestamp) throws Exception { Map entries = new HashMap<>(); @@ -511,7 +530,7 @@ public class StreamingTransferTest assertEquals(1, cfs.getSSTables().size()); assertEquals(7, Util.getRangeSlice(cfs).size()); } - + */ public interface Mutator { public void mutate(String key, String col, long timestamp) throws Exception; diff --git a/test/unit/org/apache/cassandra/streaming/compress/CompressedInputStreamTest.java b/test/unit/org/apache/cassandra/streaming/compression/CompressedInputStreamTest.java similarity index 93% rename from test/unit/org/apache/cassandra/streaming/compress/CompressedInputStreamTest.java rename to test/unit/org/apache/cassandra/streaming/compression/CompressedInputStreamTest.java index 20371c3432..c4548c65b1 100644 --- a/test/unit/org/apache/cassandra/streaming/compress/CompressedInputStreamTest.java +++ b/test/unit/org/apache/cassandra/streaming/compression/CompressedInputStreamTest.java @@ -15,18 +15,14 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -package org.apache.cassandra.streaming.compress; +package org.apache.cassandra.streaming.compression; -import java.io.ByteArrayInputStream; -import java.io.DataInputStream; -import java.io.EOFException; -import java.io.File; -import java.io.RandomAccessFile; +import java.io.*; import java.util.*; import org.junit.Test; -import org.apache.cassandra.db.composites.*; +import org.apache.cassandra.db.ClusteringComparator; import org.apache.cassandra.db.marshal.BytesType; import org.apache.cassandra.io.compress.CompressedSequentialWriter; import org.apache.cassandra.io.compress.CompressionMetadata; @@ -35,6 +31,8 @@ import org.apache.cassandra.io.compress.SnappyCompressor; import org.apache.cassandra.io.sstable.Component; import org.apache.cassandra.io.sstable.Descriptor; import org.apache.cassandra.io.sstable.metadata.MetadataCollector; +import org.apache.cassandra.streaming.compress.CompressedInputStream; +import org.apache.cassandra.streaming.compress.CompressionInfo; import org.apache.cassandra.utils.Pair; /** @@ -67,7 +65,7 @@ public class CompressedInputStreamTest // write compressed data file of longs File tmp = new File(File.createTempFile("cassandra", "unittest").getParent(), "ks-cf-ib-1-Data.db"); Descriptor desc = Descriptor.fromFilename(tmp.getAbsolutePath()); - MetadataCollector collector = new MetadataCollector(new SimpleDenseCellNameType(BytesType.instance)); + MetadataCollector collector = new MetadataCollector(new ClusteringComparator(BytesType.instance)); CompressionParameters param = new CompressionParameters(SnappyCompressor.instance, 32, Collections.EMPTY_MAP); CompressedSequentialWriter writer = new CompressedSequentialWriter(tmp, desc.filenameFor(Component.COMPRESSION_INFO), param, collector); Map index = new HashMap(); diff --git a/test/unit/org/apache/cassandra/thrift/MultiSliceTest.java b/test/unit/org/apache/cassandra/thrift/MultiSliceTest.java deleted file mode 100644 index 9716876b95..0000000000 --- a/test/unit/org/apache/cassandra/thrift/MultiSliceTest.java +++ /dev/null @@ -1,186 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.apache.cassandra.thrift; - -import java.io.IOException; -import java.net.InetSocketAddress; -import java.nio.ByteBuffer; - -import java.util.ArrayList; -import java.util.Arrays; -import java.util.List; - -import junit.framework.Assert; - -import org.apache.cassandra.SchemaLoader; -import org.apache.cassandra.config.KSMetaData; -import org.apache.cassandra.exceptions.ConfigurationException; -import org.apache.cassandra.locator.SimpleStrategy; -import org.apache.cassandra.service.EmbeddedCassandraService; -import org.apache.cassandra.utils.ByteBufferUtil; -import org.apache.thrift.TException; -import org.junit.BeforeClass; -import org.junit.Test; - -public class MultiSliceTest -{ - private static CassandraServer server; - public static final String KEYSPACE1 = "MultiSliceTest"; - public static final String CF_STANDARD = "Standard1"; - - @BeforeClass - public static void defineSchema() throws ConfigurationException, IOException, TException - { - SchemaLoader.prepareServer(); - new EmbeddedCassandraService().start(); - ThriftSessionManager.instance.setCurrentSocket(new InetSocketAddress(9160)); - SchemaLoader.createKeyspace(KEYSPACE1, - SimpleStrategy.class, - KSMetaData.optsWithRF(1), - SchemaLoader.standardCFMD(KEYSPACE1, CF_STANDARD)); - server = new CassandraServer(); - server.set_keyspace(KEYSPACE1); - } - - private static MultiSliceRequest makeMultiSliceRequest(ByteBuffer key) - { - ColumnParent cp = new ColumnParent("Standard1"); - MultiSliceRequest req = new MultiSliceRequest(); - req.setKey(key); - req.setCount(1000); - req.reversed = false; - req.setColumn_parent(cp); - return req; - } - - @Test - public void test_multi_slice_optional_column_slice() throws TException - { - ColumnParent cp = new ColumnParent("Standard1"); - ByteBuffer key = ByteBuffer.wrap("multi_slice".getBytes()); - List expected = new ArrayList(); - for (char a = 'a'; a <= 'z'; a++) - expected.add(a + ""); - - addTheAlphabetToRow(key, cp); - MultiSliceRequest req = makeMultiSliceRequest(key); - req.setColumn_slices(new ArrayList()); - req.getColumn_slices().add(new ColumnSlice()); - List list = server.get_multi_slice(req); - assertColumnNameMatches(expected, list); - } - - @Test - public void test_multi_slice() throws TException - { - ColumnParent cp = new ColumnParent("Standard1"); - ByteBuffer key = ByteBuffer.wrap("multi_slice_two_slice".getBytes()); - addTheAlphabetToRow(key, cp); - MultiSliceRequest req = makeMultiSliceRequest(key); - req.setColumn_slices(Arrays.asList(columnSliceFrom("a", "e"), columnSliceFrom("i", "n"))); - assertColumnNameMatches(Arrays.asList("a", "b", "c", "d", "e", "i", "j", "k" , "l", "m" , "n"), server.get_multi_slice(req)); - } - - @Test - public void test_with_overlap() throws TException - { - ColumnParent cp = new ColumnParent("Standard1"); - ByteBuffer key = ByteBuffer.wrap("overlap".getBytes()); - addTheAlphabetToRow(key, cp); - MultiSliceRequest req = makeMultiSliceRequest(key); - req.setColumn_slices(Arrays.asList(columnSliceFrom("a", "e"), columnSliceFrom("d", "g"))); - assertColumnNameMatches(Arrays.asList("a", "b", "c", "d", "e", "f", "g"), server.get_multi_slice(req)); - } - - @Test - public void test_with_overlap_reversed() throws TException - { - ColumnParent cp = new ColumnParent("Standard1"); - ByteBuffer key = ByteBuffer.wrap("overlap_reversed".getBytes()); - addTheAlphabetToRow(key, cp); - MultiSliceRequest req = makeMultiSliceRequest(key); - req.reversed = true; - req.setColumn_slices(Arrays.asList(columnSliceFrom("e", "a"), columnSliceFrom("g", "d"))); - assertColumnNameMatches(Arrays.asList("g", "f", "e", "d", "c", "b", "a"), server.get_multi_slice(req)); - } - - @Test(expected=InvalidRequestException.class) - public void test_that_column_slice_is_proper() throws TException - { - ByteBuffer key = ByteBuffer.wrap("overlap".getBytes()); - MultiSliceRequest req = makeMultiSliceRequest(key); - req.reversed = true; - req.setColumn_slices(Arrays.asList(columnSliceFrom("a", "e"), columnSliceFrom("g", "d"))); - assertColumnNameMatches(Arrays.asList("a", "b", "c", "d", "e", "f", "g"), server.get_multi_slice(req)); - } - - @Test - public void test_with_overlap_reversed_with_count() throws TException - { - ColumnParent cp = new ColumnParent("Standard1"); - ByteBuffer key = ByteBuffer.wrap("overlap_reversed_count".getBytes()); - addTheAlphabetToRow(key, cp); - MultiSliceRequest req = makeMultiSliceRequest(key); - req.setCount(6); - req.reversed = true; - req.setColumn_slices(Arrays.asList(columnSliceFrom("e", "a"), columnSliceFrom("g", "d"))); - assertColumnNameMatches(Arrays.asList("g", "f", "e", "d", "c", "b"), server.get_multi_slice(req)); - } - - @Test - public void test_with_overlap_with_count() throws TException - { - ColumnParent cp = new ColumnParent("Standard1"); - ByteBuffer key = ByteBuffer.wrap("overlap_reversed_count".getBytes()); - addTheAlphabetToRow(key, cp); - MultiSliceRequest req = makeMultiSliceRequest(key); - req.setCount(6); - req.setColumn_slices(Arrays.asList(columnSliceFrom("a", "e"), columnSliceFrom("d", "g"), columnSliceFrom("d", "g"))); - assertColumnNameMatches(Arrays.asList("a", "b", "c", "d", "e", "f"), server.get_multi_slice(req)); - } - - private static void addTheAlphabetToRow(ByteBuffer key, ColumnParent parent) - throws InvalidRequestException, UnavailableException, TimedOutException - { - for (char a = 'a'; a <= 'z'; a++) { - Column c1 = new Column(); - c1.setName(ByteBufferUtil.bytes(String.valueOf(a))); - c1.setValue(new byte [0]); - c1.setTimestamp(System.nanoTime()); - server.insert(key, parent, c1, ConsistencyLevel.ONE); - } - } - - private static void assertColumnNameMatches(List expected , List actual) - { - Assert.assertEquals(actual+" "+expected +" did not have same number of elements", actual.size(), expected.size()); - for (int i = 0 ; i< expected.size() ; i++) - { - Assert.assertEquals(actual.get(i) +" did not equal "+ expected.get(i), - expected.get(i), new String(actual.get(i).getColumn().getName())); - } - } - - private ColumnSlice columnSliceFrom(String startInclusive, String endInclusive) - { - ColumnSlice cs = new ColumnSlice(); - cs.setStart(ByteBufferUtil.bytes(startInclusive)); - cs.setFinish(ByteBufferUtil.bytes(endInclusive)); - return cs; - } -} diff --git a/test/unit/org/apache/cassandra/thrift/ThriftValidationTest.java b/test/unit/org/apache/cassandra/thrift/ThriftValidationTest.java deleted file mode 100644 index c693f7ccf6..0000000000 --- a/test/unit/org/apache/cassandra/thrift/ThriftValidationTest.java +++ /dev/null @@ -1,205 +0,0 @@ -package org.apache.cassandra.thrift; -/* - * - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - * - */ - -import java.nio.ByteBuffer; -import java.util.Arrays; - -import org.junit.BeforeClass; -import org.junit.Test; - -import org.apache.cassandra.SchemaLoader; -import org.apache.cassandra.config.*; -import org.apache.cassandra.db.marshal.*; -import org.apache.cassandra.exceptions.ConfigurationException; -import org.apache.cassandra.locator.LocalStrategy; -import org.apache.cassandra.locator.NetworkTopologyStrategy; -import org.apache.cassandra.locator.SimpleStrategy; -import org.apache.cassandra.utils.ByteBufferUtil; - -import static org.junit.Assert.assertEquals; -import static org.junit.Assert.assertNotNull; - -public class ThriftValidationTest -{ - public static final String KEYSPACE1 = "MultiSliceTest"; - public static final String CF_STANDARD = "Standard1"; - public static final String CF_COUNTER = "Counter1"; - public static final String CF_UUID = "UUIDKeys"; - public static final String CF_STANDARDLONG3 = "StandardLong3"; - - @BeforeClass - public static void defineSchema() throws ConfigurationException - { - SchemaLoader.prepareServer(); - SchemaLoader.createKeyspace(KEYSPACE1, - SimpleStrategy.class, - KSMetaData.optsWithRF(1), - SchemaLoader.standardCFMD(KEYSPACE1, CF_STANDARD), - SchemaLoader.standardCFMD(KEYSPACE1, CF_COUNTER).defaultValidator(CounterColumnType.instance), - SchemaLoader.standardCFMD(KEYSPACE1, CF_UUID).keyValidator(UUIDType.instance), - SchemaLoader.standardCFMD(KEYSPACE1, CF_STANDARDLONG3, IntegerType.instance)); - } - - @Test(expected=org.apache.cassandra.exceptions.InvalidRequestException.class) - public void testValidateCommutativeWithStandard() throws org.apache.cassandra.exceptions.InvalidRequestException - { - ThriftValidation.validateColumnFamily(KEYSPACE1, "Standard1", true); - } - - @Test - public void testValidateCommutativeWithCounter() throws org.apache.cassandra.exceptions.InvalidRequestException - { - ThriftValidation.validateColumnFamily(KEYSPACE1, "Counter1", true); - } - - @Test - public void testColumnNameEqualToKeyAlias() throws org.apache.cassandra.exceptions.InvalidRequestException - { - CFMetaData metaData = Schema.instance.getCFMetaData(KEYSPACE1, "Standard1"); - CFMetaData newMetadata = metaData.copy(); - - boolean gotException = false; - - // add a key_alias = "id" - // should not throw IRE here - try - { - newMetadata.addColumnDefinition(ColumnDefinition.partitionKeyDef(metaData, AsciiType.instance.decompose("id"), LongType.instance, null)); - newMetadata.validate(); - } - catch (ConfigurationException e) - { - gotException = true; - } - - assert !gotException : "got unexpected ConfigurationException"; - - - gotException = false; - - // add a column with name = "id" - try - { - newMetadata.addColumnDefinition(ColumnDefinition.regularDef(metaData, ByteBufferUtil.bytes("id"), LongType.instance, null)); - newMetadata.validate(); - } - catch (ConfigurationException e) - { - gotException = true; - } - - assert gotException : "expected ConfigurationException but not received."; - - // make sure the key alias does not affect validation of columns with the same name (CASSANDRA-6892) - Column column = new Column(ByteBufferUtil.bytes("id")); - column.setValue(ByteBufferUtil.bytes("not a long")); - column.setTimestamp(1234); - ByteBuffer key = ByteBufferUtil.bytes("key"); - ThriftValidation.validateColumnData(newMetadata, key, null, column); - } - - @Test - public void testColumnNameEqualToDefaultKeyAlias() throws org.apache.cassandra.exceptions.InvalidRequestException - { - CFMetaData metaData = Schema.instance.getCFMetaData(KEYSPACE1, "UUIDKeys"); - ColumnDefinition definition = metaData.getColumnDefinition(ByteBufferUtil.bytes(CFMetaData.DEFAULT_KEY_ALIAS)); - assertNotNull(definition); - assertEquals(ColumnDefinition.Kind.PARTITION_KEY, definition.kind); - - // make sure the key alias does not affect validation of columns with the same name (CASSANDRA-6892) - Column column = new Column(ByteBufferUtil.bytes(CFMetaData.DEFAULT_KEY_ALIAS)); - column.setValue(ByteBufferUtil.bytes("not a uuid")); - column.setTimestamp(1234); - ByteBuffer key = ByteBufferUtil.bytes("key"); - ThriftValidation.validateColumnData(metaData, key, null, column); - - IndexExpression expression = new IndexExpression(ByteBufferUtil.bytes(CFMetaData.DEFAULT_KEY_ALIAS), IndexOperator.EQ, ByteBufferUtil.bytes("a")); - ThriftValidation.validateFilterClauses(metaData, Arrays.asList(expression)); - } - - @Test - public void testColumnNameEqualToDefaultColumnAlias() throws org.apache.cassandra.exceptions.InvalidRequestException - { - CFMetaData metaData = Schema.instance.getCFMetaData(KEYSPACE1, "StandardLong3"); - ColumnDefinition definition = metaData.getColumnDefinition(ByteBufferUtil.bytes(CFMetaData.DEFAULT_COLUMN_ALIAS + 1)); - assertNotNull(definition); - - // make sure the column alias does not affect validation of columns with the same name (CASSANDRA-6892) - Column column = new Column(ByteBufferUtil.bytes(CFMetaData.DEFAULT_COLUMN_ALIAS + 1)); - column.setValue(ByteBufferUtil.bytes("not a long")); - column.setTimestamp(1234); - ByteBuffer key = ByteBufferUtil.bytes("key"); - ThriftValidation.validateColumnData(metaData, key, null, column); - } - - @Test - public void testValidateKsDef() - { - KsDef ks_def = new KsDef() - .setName("keyspaceValid") - .setStrategy_class(LocalStrategy.class.getSimpleName()); - - - boolean gotException = false; - - try - { - ThriftConversion.fromThrift(ks_def).validate(); - } - catch (ConfigurationException e) - { - gotException = true; - } - - assert gotException : "expected ConfigurationException but not received."; - - ks_def.setStrategy_class(LocalStrategy.class.getName()); - - gotException = false; - - try - { - ThriftConversion.fromThrift(ks_def).validate(); - } - catch (ConfigurationException e) - { - gotException = true; - } - - assert gotException : "expected ConfigurationException but not received."; - - ks_def.setStrategy_class(NetworkTopologyStrategy.class.getName()); - - gotException = false; - - try - { - ThriftConversion.fromThrift(ks_def).validate(); - } - catch (ConfigurationException e) - { - gotException = true; - } - - assert !gotException : "got unexpected ConfigurationException"; - } -} diff --git a/test/unit/org/apache/cassandra/tools/SSTableExportTest.java b/test/unit/org/apache/cassandra/tools/SSTableExportTest.java deleted file mode 100644 index bc73c8302b..0000000000 --- a/test/unit/org/apache/cassandra/tools/SSTableExportTest.java +++ /dev/null @@ -1,405 +0,0 @@ -/* -* Licensed to the Apache Software Foundation (ASF) under one -* or more contributor license agreements. See the NOTICE file -* distributed with this work for additional information -* regarding copyright ownership. The ASF licenses this file -* to you under the Apache License, Version 2.0 (the -* "License"); you may not use this file except in compliance -* with the License. You may obtain a copy of the License at -* -* http://www.apache.org/licenses/LICENSE-2.0 -* -* Unless required by applicable law or agreed to in writing, -* software distributed under the License is distributed on an -* "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -* KIND, either express or implied. See the License for the -* specific language governing permissions and limitations -* under the License. -*/ -package org.apache.cassandra.tools; - -import java.io.File; -import java.io.FileReader; -import java.io.IOException; -import java.io.PrintStream; - -import org.junit.BeforeClass; -import org.junit.Test; - -import org.apache.cassandra.SchemaLoader; -import org.apache.cassandra.Util; -import org.apache.cassandra.config.CFMetaData; -import org.apache.cassandra.config.KSMetaData; -import org.apache.cassandra.db.ArrayBackedSortedColumns; -import org.apache.cassandra.db.BufferCell; -import org.apache.cassandra.db.BufferCounterCell; -import org.apache.cassandra.db.BufferExpiringCell; -import org.apache.cassandra.db.ColumnFamily; -import org.apache.cassandra.db.ColumnFamilyStore; -import org.apache.cassandra.db.DeletionInfo; -import org.apache.cassandra.db.Keyspace; -import org.apache.cassandra.db.filter.QueryFilter; -import org.apache.cassandra.db.marshal.AsciiType; -import org.apache.cassandra.db.marshal.BytesType; -import org.apache.cassandra.db.marshal.CounterColumnType; -import org.apache.cassandra.db.marshal.UTF8Type; -import org.apache.cassandra.db.marshal.UUIDType; -import org.apache.cassandra.exceptions.ConfigurationException; -import org.apache.cassandra.io.sstable.Descriptor; -import org.apache.cassandra.io.sstable.format.SSTableReader; -import org.apache.cassandra.io.sstable.format.SSTableWriter; -import org.apache.cassandra.locator.SimpleStrategy; -import org.apache.cassandra.service.ActiveRepairService; -import org.apache.cassandra.utils.ByteBufferUtil; -import org.apache.cassandra.utils.UUIDGen; -import org.apache.thrift.TException; -import org.json.simple.JSONArray; -import org.json.simple.JSONObject; -import org.json.simple.JSONValue; -import org.json.simple.parser.ParseException; - -import static org.apache.cassandra.Util.column; -import static org.apache.cassandra.io.sstable.SSTableUtils.tempSSTableFile; -import static org.apache.cassandra.utils.ByteBufferUtil.bytesToHex; -import static org.apache.cassandra.utils.ByteBufferUtil.hexToBytes; -import static org.junit.Assert.assertEquals; -import static org.junit.Assert.assertNotNull; - -public class SSTableExportTest -{ - public static final String KEYSPACE1 = "SSTableExportTest"; - public static final String CF_STANDARD = "Standard1"; - public static final String CF_COUNTER = "Counter1"; - public static final String CF_UUID = "UUIDKeys"; - public static final String CF_VALSWITHQUOTES = "ValuesWithQuotes"; - - @BeforeClass - public static void defineSchema() throws ConfigurationException - { - SchemaLoader.prepareServer(); - SchemaLoader.createKeyspace(KEYSPACE1, - SimpleStrategy.class, - KSMetaData.optsWithRF(1), - SchemaLoader.standardCFMD(KEYSPACE1, CF_STANDARD), - SchemaLoader.standardCFMD(KEYSPACE1, CF_COUNTER).defaultValidator(CounterColumnType.instance), - SchemaLoader.standardCFMD(KEYSPACE1, CF_UUID).keyValidator(UUIDType.instance), - SchemaLoader.standardCFMD(KEYSPACE1, CF_VALSWITHQUOTES).defaultValidator(UTF8Type.instance), - SchemaLoader.standardCFMD(KEYSPACE1, "AsciiKeys").keyValidator(AsciiType.instance)); - } - - public String asHex(String str) - { - return bytesToHex(ByteBufferUtil.bytes(str)); - } - - @Test - public void testEnumeratekeys() throws IOException - { - File tempSS = tempSSTableFile(KEYSPACE1, "Standard1"); - ColumnFamily cfamily = ArrayBackedSortedColumns.factory.create(KEYSPACE1, "Standard1"); - SSTableWriter writer = SSTableWriter.create(tempSS.getPath(), 2, ActiveRepairService.UNREPAIRED_SSTABLE, 0); - - // Add rowA - cfamily.addColumn(Util.cellname("colA"), ByteBufferUtil.bytes("valA"), System.currentTimeMillis()); - writer.append(Util.dk("rowA"), cfamily); - cfamily.clear(); - - // Add rowB - cfamily.addColumn(Util.cellname("colB"), ByteBufferUtil.bytes("valB"), System.currentTimeMillis()); - writer.append(Util.dk("rowB"), cfamily); - cfamily.clear(); - - writer.finish(true); - - // Enumerate and verify - File temp = File.createTempFile("Standard1", ".txt"); - final Descriptor descriptor = Descriptor.fromFilename(writer.getFilename()); - SSTableExport.enumeratekeys(descriptor, new PrintStream(temp.getPath()), - CFMetaData.sparseCFMetaData(descriptor.ksname, descriptor.cfname, BytesType.instance)); - - - try (FileReader file = new FileReader(temp)) - { - char[] buf = new char[(int) temp.length()]; - file.read(buf); - String output = new String(buf); - - String sep = System.getProperty("line.separator"); - assert output.equals(asHex("rowA") + sep + asHex("rowB") + sep) : output; - } - } - - @Test - public void testExportSimpleCf() throws IOException, ParseException - { - File tempSS = tempSSTableFile(KEYSPACE1, "Standard1"); - ColumnFamily cfamily = ArrayBackedSortedColumns.factory.create(KEYSPACE1, "Standard1"); - SSTableWriter writer = SSTableWriter.create(tempSS.getPath(), 2, ActiveRepairService.UNREPAIRED_SSTABLE, 0); - - int nowInSec = (int)(System.currentTimeMillis() / 1000) + 42; //live for 42 seconds - // Add rowA - cfamily.addColumn(Util.cellname("colA"), ByteBufferUtil.bytes("valA"), System.currentTimeMillis()); - cfamily.addColumn(new BufferExpiringCell(Util.cellname("colExp"), ByteBufferUtil.bytes("valExp"), System.currentTimeMillis(), 42, nowInSec)); - writer.append(Util.dk("rowA"), cfamily); - cfamily.clear(); - - // Add rowB - cfamily.addColumn(Util.cellname("colB"), ByteBufferUtil.bytes("valB"), System.currentTimeMillis()); - writer.append(Util.dk("rowB"), cfamily); - cfamily.clear(); - - // Add rowExclude - cfamily.addColumn(Util.cellname("colX"), ByteBufferUtil.bytes("valX"), System.currentTimeMillis()); - writer.append(Util.dk("rowExclude"), cfamily); - cfamily.clear(); - - SSTableReader reader = writer.finish(true); - - // Export to JSON and verify - File tempJson = File.createTempFile("Standard1", ".json"); - SSTableExport.export(reader, new PrintStream(tempJson.getPath()), new String[]{asHex("rowExclude")}); - - JSONArray json = (JSONArray)JSONValue.parseWithException(new FileReader(tempJson)); - assertEquals("unexpected number of rows", 2, json.size()); - - JSONObject rowA = (JSONObject)json.get(0); - assertEquals("unexpected number of keys", 2, rowA.keySet().size()); - assertEquals("unexpected row key",asHex("rowA"),rowA.get("key")); - - JSONArray colsA = (JSONArray)rowA.get("cells"); - JSONArray colA = (JSONArray)colsA.get(0); - assert hexToBytes((String)colA.get(1)).equals(ByteBufferUtil.bytes("valA")); - - JSONArray colExp = (JSONArray)colsA.get(1); - assert ((Long)colExp.get(4)) == 42; - assert ((Long)colExp.get(5)) == nowInSec; - - JSONObject rowB = (JSONObject)json.get(1); - assertEquals("unexpected number of keys", 2, rowB.keySet().size()); - assertEquals("unexpected row key",asHex("rowB"),rowB.get("key")); - - JSONArray colsB = (JSONArray)rowB.get("cells"); - JSONArray colB = (JSONArray)colsB.get(0); - assert colB.size() == 3; - - } - - @Test - public void testRoundTripStandardCf() throws IOException - { - ColumnFamilyStore cfs = Keyspace.open(KEYSPACE1).getColumnFamilyStore("Standard1"); - File tempSS = tempSSTableFile(KEYSPACE1, "Standard1"); - ColumnFamily cfamily = ArrayBackedSortedColumns.factory.create(KEYSPACE1, "Standard1"); - SSTableWriter writer = SSTableWriter.create(tempSS.getPath(), 2, ActiveRepairService.UNREPAIRED_SSTABLE, 0); - - // Add rowA - cfamily.addColumn(Util.cellname("name"), ByteBufferUtil.bytes("val"), System.currentTimeMillis()); - writer.append(Util.dk("rowA"), cfamily); - cfamily.clear(); - - // Add rowExclude - cfamily.addColumn(Util.cellname("name"), ByteBufferUtil.bytes("val"), System.currentTimeMillis()); - writer.append(Util.dk("rowExclude"), cfamily); - cfamily.clear(); - - SSTableReader reader = writer.finish(true); - - // Export to JSON and verify - File tempJson = File.createTempFile("Standard1", ".json"); - SSTableExport.export(reader, new PrintStream(tempJson.getPath()), new String[]{asHex("rowExclude")}); - - // Import JSON to another SSTable file - File tempSS2 = tempSSTableFile(KEYSPACE1, "Standard1"); - new SSTableImport().importJson(tempJson.getPath(), KEYSPACE1, "Standard1", tempSS2.getPath()); - - reader = SSTableReader.open(Descriptor.fromFilename(tempSS2.getPath())); - QueryFilter qf = Util.namesQueryFilter(cfs, Util.dk("rowA"), "name"); - ColumnFamily cf = qf.getSSTableColumnIterator(reader).getColumnFamily(); - qf.collateOnDiskAtom(cf, qf.getSSTableColumnIterator(reader), Integer.MIN_VALUE); - assertNotNull(cf); - assertEquals(hexToBytes("76616c"), cf.getColumn(Util.cellname("name")).value()); - - qf = Util.namesQueryFilter(cfs, Util.dk("rowExclude"), "name"); - cf = qf.getSSTableColumnIterator(reader).getColumnFamily(); - assert cf == null; - reader.selfRef().release(); - } - - @Test - public void testExportCounterCf() throws IOException, ParseException - { - File tempSS = tempSSTableFile(KEYSPACE1, "Counter1"); - ColumnFamily cfamily = ArrayBackedSortedColumns.factory.create(KEYSPACE1, "Counter1"); - SSTableWriter writer = SSTableWriter.create(tempSS.getPath(), 2, ActiveRepairService.UNREPAIRED_SSTABLE, 0); - - // Add rowA - cfamily.addColumn(BufferCounterCell.createLocal(Util.cellname("colA"), 42, System.currentTimeMillis(), Long.MIN_VALUE)); - writer.append(Util.dk("rowA"), cfamily); - cfamily.clear(); - - SSTableReader reader = writer.finish(true); - - // Export to JSON and verify - File tempJson = File.createTempFile("Counter1", ".json"); - SSTableExport.export(reader, new PrintStream(tempJson.getPath()), new String[0]); - JSONArray json = (JSONArray)JSONValue.parseWithException(new FileReader(tempJson)); - assertEquals("unexpected number of rows", 1, json.size()); - - JSONObject row = (JSONObject)json.get(0); - assertEquals("unexpected number of keys", 2, row.keySet().size()); - assertEquals("unexpected row key",asHex("rowA"),row.get("key")); - - JSONArray cols = (JSONArray)row.get("cells"); - JSONArray colA = (JSONArray)cols.get(0); - assert hexToBytes((String)colA.get(0)).equals(ByteBufferUtil.bytes("colA")); - assert ((String) colA.get(3)).equals("c"); - assert (Long) colA.get(4) == Long.MIN_VALUE; - } - - @Test - public void testEscapingDoubleQuotes() throws IOException, ParseException - { - File tempSS = tempSSTableFile(KEYSPACE1, "ValuesWithQuotes"); - ColumnFamily cfamily = ArrayBackedSortedColumns.factory.create(KEYSPACE1, "ValuesWithQuotes"); - SSTableWriter writer = SSTableWriter.create(tempSS.getPath(), 2, ActiveRepairService.UNREPAIRED_SSTABLE); - - // Add rowA - cfamily.addColumn(new BufferCell(Util.cellname("data"), UTF8Type.instance.fromString("{\"foo\":\"bar\"}"))); - writer.append(Util.dk("rowA"), cfamily); - cfamily.clear(); - - SSTableReader reader = writer.finish(true); - - // Export to JSON and verify - File tempJson = File.createTempFile("ValuesWithQuotes", ".json"); - SSTableExport.export(reader, new PrintStream(tempJson.getPath()), new String[0]); - - JSONArray json = (JSONArray)JSONValue.parseWithException(new FileReader(tempJson)); - assertEquals("unexpected number of rows", 1, json.size()); - - JSONObject row = (JSONObject)json.get(0); - assertEquals("unexpected number of keys", 2, row.keySet().size()); - assertEquals("unexpected row key",asHex("rowA"),row.get("key")); - - JSONArray cols = (JSONArray)row.get("cells"); - JSONArray colA = (JSONArray)cols.get(0); - assert hexToBytes((String)colA.get(0)).equals(ByteBufferUtil.bytes("data")); - assert colA.get(1).equals("{\"foo\":\"bar\"}"); - } - - @Test - public void testExportColumnsWithMetadata() throws IOException, ParseException - { - File tempSS = tempSSTableFile(KEYSPACE1, "Standard1"); - ColumnFamily cfamily = ArrayBackedSortedColumns.factory.create(KEYSPACE1, "Standard1"); - SSTableWriter writer = SSTableWriter.create(tempSS.getPath(), 2, ActiveRepairService.UNREPAIRED_SSTABLE); - - // Add rowA - cfamily.addColumn(Util.cellname("colName"), ByteBufferUtil.bytes("val"), System.currentTimeMillis()); - cfamily.addColumn(Util.cellname("colName1"), ByteBufferUtil.bytes("val1"), System.currentTimeMillis()); - cfamily.delete(new DeletionInfo(0, 0)); - writer.append(Util.dk("rowA"), cfamily); - - SSTableReader reader = writer.finish(true); - // Export to JSON and verify - File tempJson = File.createTempFile("CFWithDeletionInfo", ".json"); - SSTableExport.export(reader, new PrintStream(tempJson.getPath()), new String[0]); - - JSONArray json = (JSONArray)JSONValue.parseWithException(new FileReader(tempJson)); - assertEquals("unexpected number of rows", 1, json.size()); - - JSONObject row = (JSONObject)json.get(0); - assertEquals("unexpected number of keys", 3, row.keySet().size()); - assertEquals("unexpected row key",asHex("rowA"),row.get("key")); - - // check that the row key is there and present - String rowKey = (String) row.get("key"); - assertNotNull("expecing key to be present", rowKey); - assertEquals("key did not match", ByteBufferUtil.bytes("rowA"), hexToBytes(rowKey)); - - // check that there is metadata and that it contains deletionInfo - JSONObject meta = (JSONObject) row.get("metadata"); - assertNotNull("expecing metadata to be present", meta); - - assertEquals("unexpected number of metadata entries", 1, meta.keySet().size()); - - JSONObject serializedDeletionInfo = (JSONObject) meta.get("deletionInfo"); - assertNotNull("expecing deletionInfo to be present", serializedDeletionInfo); - - assertEquals( - "unexpected serialization format for topLevelDeletion", - JSONValue.parse("{\"markedForDeleteAt\":0,\"localDeletionTime\":0}"), - serializedDeletionInfo); - - // check the colums are what we put in - JSONArray cols = (JSONArray) row.get("cells"); - assertNotNull("expecing columns to be present", cols); - assertEquals("expecting two columns", 2, cols.size()); - - JSONArray col1 = (JSONArray) cols.get(0); - assertEquals("column name did not match", ByteBufferUtil.bytes("colName"), hexToBytes((String) col1.get(0))); - assertEquals("column value did not match", ByteBufferUtil.bytes("val"), hexToBytes((String) col1.get(1))); - - JSONArray col2 = (JSONArray) cols.get(1); - assertEquals("column name did not match", ByteBufferUtil.bytes("colName1"), hexToBytes((String) col2.get(0))); - assertEquals("column value did not match", ByteBufferUtil.bytes("val1"), hexToBytes((String) col2.get(1))); - } - - /** - * Tests CASSANDRA-6892 (key aliases being used improperly for validation) - */ - @Test - public void testColumnNameEqualToDefaultKeyAlias() throws IOException, ParseException - { - File tempSS = tempSSTableFile(KEYSPACE1, "UUIDKeys"); - ColumnFamily cfamily = ArrayBackedSortedColumns.factory.create(KEYSPACE1, "UUIDKeys"); - SSTableWriter writer = SSTableWriter.create(tempSS.getPath(), 2, ActiveRepairService.UNREPAIRED_SSTABLE); - - // Add a row - cfamily.addColumn(column(CFMetaData.DEFAULT_KEY_ALIAS, "not a uuid", 1L)); - writer.append(Util.dk(ByteBufferUtil.bytes(UUIDGen.getTimeUUID())), cfamily); - - SSTableReader reader = writer.finish(true); - // Export to JSON and verify - File tempJson = File.createTempFile("CFWithColumnNameEqualToDefaultKeyAlias", ".json"); - SSTableExport.export(reader, new PrintStream(tempJson.getPath()), new String[0]); - - JSONArray json = (JSONArray)JSONValue.parseWithException(new FileReader(tempJson)); - assertEquals(1, json.size()); - - JSONObject row = (JSONObject)json.get(0); - JSONArray cols = (JSONArray) row.get("cells"); - assertEquals(1, cols.size()); - - // check column name and value - JSONArray col = (JSONArray) cols.get(0); - assertEquals(CFMetaData.DEFAULT_KEY_ALIAS, ByteBufferUtil.string(hexToBytes((String) col.get(0)))); - assertEquals("not a uuid", ByteBufferUtil.string(hexToBytes((String) col.get(1)))); - } - - @Test - public void testAsciiKeyValidator() throws IOException, ParseException - { - File tempSS = tempSSTableFile(KEYSPACE1, "AsciiKeys"); - ColumnFamily cfamily = ArrayBackedSortedColumns.factory.create(KEYSPACE1, "AsciiKeys"); - SSTableWriter writer = SSTableWriter.create(tempSS.getPath(), 2, ActiveRepairService.UNREPAIRED_SSTABLE, 0); - - // Add a row - cfamily.addColumn(column("column", "value", 1L)); - writer.append(Util.dk("key", AsciiType.instance), cfamily); - - SSTableReader reader = writer.finish(true); - // Export to JSON and verify - File tempJson = File.createTempFile("CFWithAsciiKeys", ".json"); - SSTableExport.export(reader, - new PrintStream(tempJson.getPath()), - new String[0]); - - JSONArray json = (JSONArray)JSONValue.parseWithException(new FileReader(tempJson)); - assertEquals(1, json.size()); - - JSONObject row = (JSONObject)json.get(0); - // check row key - assertEquals("key", row.get("key")); - } -} diff --git a/test/unit/org/apache/cassandra/tools/SSTableImportTest.java b/test/unit/org/apache/cassandra/tools/SSTableImportTest.java deleted file mode 100644 index 814662c87d..0000000000 --- a/test/unit/org/apache/cassandra/tools/SSTableImportTest.java +++ /dev/null @@ -1,278 +0,0 @@ -/* -* Licensed to the Apache Software Foundation (ASF) under one -* or more contributor license agreements. See the NOTICE file -* distributed with this work for additional information -* regarding copyright ownership. The ASF licenses this file -* to you under the Apache License, Version 2.0 (the -* "License"); you may not use this file except in compliance -* with the License. You may obtain a copy of the License at -* -* http://www.apache.org/licenses/LICENSE-2.0 -* -* Unless required by applicable law or agreed to in writing, -* software distributed under the License is distributed on an -* "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -* KIND, either express or implied. See the License for the -* specific language governing permissions and limitations -* under the License. -*/ -package org.apache.cassandra.tools; - -import static org.junit.Assert.assertEquals; -import static org.hamcrest.CoreMatchers.is; -import static org.junit.Assert.assertThat; -import static org.junit.matchers.JUnitMatchers.hasItem; - -import static org.apache.cassandra.io.sstable.SSTableUtils.tempSSTableFile; -import static org.apache.cassandra.utils.ByteBufferUtil.hexToBytes; - -import java.io.File; -import java.io.IOException; -import java.net.URI; -import java.net.URISyntaxException; - -import org.apache.cassandra.io.sstable.format.SSTableReader; -import org.junit.BeforeClass; -import org.hamcrest.Description; -import org.hamcrest.Matcher; -import org.junit.Test; -import org.junit.internal.matchers.TypeSafeMatcher; - -import org.apache.cassandra.SchemaLoader; -import org.apache.cassandra.Util; -import org.apache.cassandra.db.*; -import org.apache.cassandra.config.CFMetaData; -import org.apache.cassandra.config.KSMetaData; -import org.apache.cassandra.db.ArrayBackedSortedColumns; -import org.apache.cassandra.db.BufferDeletedCell; -import org.apache.cassandra.db.Cell; -import org.apache.cassandra.db.ColumnFamily; -import org.apache.cassandra.db.CounterCell; -import org.apache.cassandra.db.DeletionInfo; -import org.apache.cassandra.db.ExpiringCell; -import org.apache.cassandra.cql3.QueryProcessor; -import org.apache.cassandra.cql3.UntypedResultSet; -import org.apache.cassandra.cql3.UntypedResultSet.Row; -import org.apache.cassandra.db.columniterator.OnDiskAtomIterator; -import org.apache.cassandra.db.filter.QueryFilter; -import org.apache.cassandra.db.marshal.AsciiType; -import org.apache.cassandra.db.marshal.BytesType; -import org.apache.cassandra.db.marshal.CounterColumnType; -import org.apache.cassandra.exceptions.ConfigurationException; -import org.apache.cassandra.io.sstable.Descriptor; -import org.apache.cassandra.locator.SimpleStrategy; -import org.apache.thrift.TException; - -public class SSTableImportTest -{ - public static final String KEYSPACE1 = "SSTableImportTest"; - public static final String CF_STANDARD = "Standard1"; - public static final String CF_COUNTER = "Counter1"; - public static final String CQL_TABLE = "table1"; - - @BeforeClass - public static void defineSchema() throws ConfigurationException - { - SchemaLoader.prepareServer(); - SchemaLoader.createKeyspace(KEYSPACE1, - SimpleStrategy.class, - KSMetaData.optsWithRF(1), - SchemaLoader.standardCFMD(KEYSPACE1, CF_STANDARD), - SchemaLoader.standardCFMD(KEYSPACE1, CF_COUNTER).defaultValidator(CounterColumnType.instance), - SchemaLoader.standardCFMD(KEYSPACE1, "AsciiKeys").keyValidator(AsciiType.instance), - CFMetaData.compile("CREATE TABLE table1 (k int PRIMARY KEY, v1 text, v2 int)", KEYSPACE1)); - } - - @Test - public void testImportSimpleCf() throws IOException, URISyntaxException - { - // Import JSON to temp SSTable file - String jsonUrl = resourcePath("SimpleCF.json"); - File tempSS = tempSSTableFile(KEYSPACE1, "Standard1"); - new SSTableImport(true).importJson(jsonUrl, KEYSPACE1, "Standard1", tempSS.getPath()); - - // Verify results - SSTableReader reader = SSTableReader.open(Descriptor.fromFilename(tempSS.getPath())); - QueryFilter qf = QueryFilter.getIdentityFilter(Util.dk("rowA"), "Standard1", System.currentTimeMillis()); - OnDiskAtomIterator iter = qf.getSSTableColumnIterator(reader); - ColumnFamily cf = cloneForAdditions(iter); - while (iter.hasNext()) cf.addAtom(iter.next()); - assert cf.getColumn(Util.cellname("colAA")).value().equals(hexToBytes("76616c4141")); - assert !(cf.getColumn(Util.cellname("colAA")) instanceof BufferDeletedCell); - Cell expCol = cf.getColumn(Util.cellname("colAC")); - assert expCol.value().equals(hexToBytes("76616c4143")); - assert expCol instanceof ExpiringCell; - assert ((ExpiringCell)expCol).getTimeToLive() == 42 && expCol.getLocalDeletionTime() == 2000000000; - reader.selfRef().release(); - } - - private ColumnFamily cloneForAdditions(OnDiskAtomIterator iter) - { - return iter.getColumnFamily().cloneMeShallow(ArrayBackedSortedColumns.factory, false); - } - - private String resourcePath(String name) throws URISyntaxException - { - // Naive resource.getPath fails on Windows in many cases, for example if there are spaces in the path - // which get encoded as %20 which Windows doesn't like. The trick is to create a URI first, which satisfies all platforms. - return new URI(getClass().getClassLoader().getResource(name).toString()).getPath(); - } - - @Test - public void testImportUnsortedMode() throws IOException, URISyntaxException - { - String jsonUrl = resourcePath("UnsortedCF.json"); - File tempSS = tempSSTableFile(KEYSPACE1, "Standard1"); - - new SSTableImport().importJson(jsonUrl, KEYSPACE1, "Standard1", tempSS.getPath()); - - SSTableReader reader = SSTableReader.open(Descriptor.fromFilename(tempSS.getPath())); - QueryFilter qf = QueryFilter.getIdentityFilter(Util.dk("rowA"), "Standard1", System.currentTimeMillis()); - OnDiskAtomIterator iter = qf.getSSTableColumnIterator(reader); - ColumnFamily cf = cloneForAdditions(iter); - while (iter.hasNext()) - cf.addAtom(iter.next()); - assert cf.getColumn(Util.cellname("colAA")).value().equals(hexToBytes("76616c4141")); - assert !(cf.getColumn(Util.cellname("colAA")) instanceof BufferDeletedCell); - Cell expCol = cf.getColumn(Util.cellname("colAC")); - assert expCol.value().equals(hexToBytes("76616c4143")); - assert expCol instanceof ExpiringCell; - assert ((ExpiringCell) expCol).getTimeToLive() == 42 && expCol.getLocalDeletionTime() == 2000000000; - reader.selfRef().release(); - } - - @Test - public void testImportWithDeletionInfoMetadata() throws IOException, URISyntaxException - { - // Import JSON to temp SSTable file - String jsonUrl = resourcePath("SimpleCFWithDeletionInfo.json"); - File tempSS = tempSSTableFile(KEYSPACE1, "Standard1"); - new SSTableImport(true).importJson(jsonUrl, KEYSPACE1, "Standard1", tempSS.getPath()); - - // Verify results - SSTableReader reader = SSTableReader.open(Descriptor.fromFilename(tempSS.getPath())); - QueryFilter qf = QueryFilter.getIdentityFilter(Util.dk("rowA"), "Standard1", System.currentTimeMillis()); - OnDiskAtomIterator iter = qf.getSSTableColumnIterator(reader); - ColumnFamily cf = cloneForAdditions(iter); - assertEquals(cf.deletionInfo(), new DeletionInfo(0, 0)); - while (iter.hasNext()) - cf.addAtom(iter.next()); - assert cf.getColumn(Util.cellname("colAA")).value().equals(hexToBytes("76616c4141")); - assert !(cf.getColumn(Util.cellname("colAA")) instanceof BufferDeletedCell); - Cell expCol = cf.getColumn(Util.cellname("colAC")); - assert expCol.value().equals(hexToBytes("76616c4143")); - assert expCol instanceof ExpiringCell; - assert ((ExpiringCell) expCol).getTimeToLive() == 42 && expCol.getLocalDeletionTime() == 2000000000; - reader.selfRef().release(); - } - - @Test - public void testImportCounterCf() throws IOException, URISyntaxException - { - // Import JSON to temp SSTable file - String jsonUrl = resourcePath("CounterCF.json"); - File tempSS = tempSSTableFile(KEYSPACE1, "Counter1"); - new SSTableImport(true).importJson(jsonUrl, KEYSPACE1, "Counter1", tempSS.getPath()); - - // Verify results - SSTableReader reader = SSTableReader.open(Descriptor.fromFilename(tempSS.getPath())); - QueryFilter qf = QueryFilter.getIdentityFilter(Util.dk("rowA"), "Counter1", System.currentTimeMillis()); - OnDiskAtomIterator iter = qf.getSSTableColumnIterator(reader); - ColumnFamily cf = cloneForAdditions(iter); - while (iter.hasNext()) cf.addAtom(iter.next()); - Cell c = cf.getColumn(Util.cellname("colAA")); - assert c instanceof CounterCell : c; - assert ((CounterCell) c).total() == 42; - reader.selfRef().release(); - } - - @Test - public void testImportWithAsciiKeyValidator() throws IOException, URISyntaxException - { - // Import JSON to temp SSTable file - String jsonUrl = resourcePath("SimpleCF.json"); - File tempSS = tempSSTableFile(KEYSPACE1, "AsciiKeys"); - System.setProperty("skip.key.validator", "false"); - new SSTableImport(true).importJson(jsonUrl, KEYSPACE1, "AsciiKeys", tempSS.getPath()); - - // Verify results - SSTableReader reader = SSTableReader.open(Descriptor.fromFilename(tempSS.getPath())); - // check that keys are treated as ascii - QueryFilter qf = QueryFilter.getIdentityFilter(Util.dk("726f7741", AsciiType.instance), "AsciiKeys", System.currentTimeMillis()); - OnDiskAtomIterator iter = qf.getSSTableColumnIterator(reader); - assert iter.hasNext(); // "ascii" key exists - QueryFilter qf2 = QueryFilter.getIdentityFilter(Util.dk("726f7741", BytesType.instance), "AsciiKeys", System.currentTimeMillis()); - OnDiskAtomIterator iter2 = qf2.getSSTableColumnIterator(reader); - assert !iter2.hasNext(); // "bytes" key does not exist - reader.selfRef().release(); - } - - @Test - public void testBackwardCompatibilityOfImportWithAsciiKeyValidator() throws IOException, URISyntaxException - { - // Import JSON to temp SSTable file - String jsonUrl = resourcePath("SimpleCF.json"); - File tempSS = tempSSTableFile(KEYSPACE1, "AsciiKeys"); - // To ignore current key validator - System.setProperty("skip.key.validator", "true"); - new SSTableImport(true).importJson(jsonUrl, KEYSPACE1, "AsciiKeys", tempSS.getPath()); - - // Verify results - SSTableReader reader = SSTableReader.open(Descriptor.fromFilename(tempSS.getPath())); - // check that keys are treated as bytes - QueryFilter qf = QueryFilter.getIdentityFilter(Util.dk("rowA"), "AsciiKeys", System.currentTimeMillis()); - OnDiskAtomIterator iter = qf.getSSTableColumnIterator(reader); - assert iter.hasNext(); // "bytes" key exists - reader.selfRef().release(); - } - - @Test - /* - * The schema is - * CREATE TABLE cql_keyspace.table1 (k int PRIMARY KEY, v1 text, v2 int) - * */ - public void shouldImportCqlTable() throws IOException, URISyntaxException - { - String jsonUrl = resourcePath("CQLTable.json"); - File tempSS = tempSSTableFile(KEYSPACE1, CQL_TABLE); - new SSTableImport(true).importJson(jsonUrl, KEYSPACE1, CQL_TABLE, tempSS.getPath()); - SSTableReader reader = SSTableReader.open(Descriptor.fromFilename(tempSS.getPath())); - Keyspace.open(KEYSPACE1).getColumnFamilyStore(CQL_TABLE).addSSTable(reader); - - UntypedResultSet result = QueryProcessor.executeOnceInternal(String.format("SELECT * FROM \"%s\".%s", KEYSPACE1, CQL_TABLE)); - assertThat(result.size(), is(2)); - assertThat(result, hasItem(withElements(1, "NY", 1980))); - assertThat(result, hasItem(withElements(2, "CA", 2014))); - reader.selfRef().release(); - } - - @Test(expected=AssertionError.class) - public void shouldRejectEmptyCellNamesForNonCqlTables() throws IOException, URISyntaxException - { - String jsonUrl = resourcePath("CQLTable.json"); - File tempSS = tempSSTableFile("Keyspace1", "Counter1"); - new SSTableImport(true).importJson(jsonUrl, "Keyspace1", "Counter1", tempSS.getPath()); - } - - private static Matcher withElements(final int key, final String v1, final int v2) { - return new TypeSafeMatcher() - { - @Override - public boolean matchesSafely(Row input) - { - if (!input.has("k") || !input.has("v1") || !input.has("v2")) - return false; - return input.getInt("k") == key - && input.getString("v1").equals(v1) - && input.getInt("v2") == v2; - } - - @Override - public void describeTo(Description description) - { - description.appendText(String.format("a row containing: %s, %s, %s", key, v1, v2)); - } - }; - - } -} diff --git a/test/unit/org/apache/cassandra/transport/MessagePayloadTest.java b/test/unit/org/apache/cassandra/transport/MessagePayloadTest.java index 1dd3c5da81..865a17370d 100644 --- a/test/unit/org/apache/cassandra/transport/MessagePayloadTest.java +++ b/test/unit/org/apache/cassandra/transport/MessagePayloadTest.java @@ -65,8 +65,6 @@ public class MessagePayloadTest extends CQLTester { try { - DatabaseDescriptor.setPartitioner(ByteOrderedPartitioner.instance); - cqlQueryHandlerField = ClientState.class.getDeclaredField("cqlQueryHandler"); cqlQueryHandlerField.setAccessible(true); diff --git a/test/unit/org/apache/cassandra/triggers/TriggerExecutorTest.java b/test/unit/org/apache/cassandra/triggers/TriggerExecutorTest.java index 3d505c8408..a5d79d3219 100644 --- a/test/unit/org/apache/cassandra/triggers/TriggerExecutorTest.java +++ b/test/unit/org/apache/cassandra/triggers/TriggerExecutorTest.java @@ -17,23 +17,29 @@ */ package org.apache.cassandra.triggers; -import java.nio.ByteBuffer; import java.util.*; + import org.junit.Test; +import org.apache.cassandra.Util; import org.apache.cassandra.config.CFMetaData; -import org.apache.cassandra.config.ColumnDefinition; import org.apache.cassandra.config.TriggerDefinition; import org.apache.cassandra.db.*; -import org.apache.cassandra.db.composites.CellName; -import org.apache.cassandra.db.marshal.CompositeType; +import org.apache.cassandra.db.rows.UnfilteredRowIterators; +import org.apache.cassandra.db.rows.Cell; +import org.apache.cassandra.db.rows.Row; +import org.apache.cassandra.db.rows.RowIterator; import org.apache.cassandra.db.marshal.UTF8Type; +import org.apache.cassandra.db.partitions.Partition; +import org.apache.cassandra.db.partitions.PartitionUpdate; import org.apache.cassandra.exceptions.ConfigurationException; import org.apache.cassandra.exceptions.InvalidRequestException; - -import static org.junit.Assert.*; +import org.apache.cassandra.utils.FBUtilities; import static org.apache.cassandra.utils.ByteBufferUtil.bytes; +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertNull; +import static org.junit.Assert.assertTrue; public class TriggerExecutorTest { @@ -41,30 +47,35 @@ public class TriggerExecutorTest public void sameKeySameCfColumnFamilies() throws ConfigurationException, InvalidRequestException { CFMetaData metadata = makeCfMetaData("ks1", "cf1", TriggerDefinition.create("test", SameKeySameCfTrigger.class.getName())); - ColumnFamily mutated = TriggerExecutor.instance.execute(bytes("k1"), makeCf(metadata, "v1", null)); - assertEquals(bytes("v1"), mutated.getColumn(getColumnName(metadata, "c1")).value()); - assertEquals(bytes("trigger"), mutated.getColumn(getColumnName(metadata, "c2")).value()); + PartitionUpdate mutated = TriggerExecutor.instance.execute(makeCf(metadata, "k1", "v1", null)); + + RowIterator rowIterator = UnfilteredRowIterators.filter(mutated.unfilteredIterator(), FBUtilities.nowInSeconds()); + + Iterator cells = rowIterator.next().iterator(); + assertEquals(bytes("trigger"), cells.next().value()); + + assertTrue(!rowIterator.hasNext()); } @Test(expected = InvalidRequestException.class) public void sameKeyDifferentCfColumnFamilies() throws ConfigurationException, InvalidRequestException { CFMetaData metadata = makeCfMetaData("ks1", "cf1", TriggerDefinition.create("test", SameKeyDifferentCfTrigger.class.getName())); - TriggerExecutor.instance.execute(bytes("k1"), makeCf(metadata, "v1", null)); + TriggerExecutor.instance.execute(makeCf(metadata, "k1", "v1", null)); } @Test(expected = InvalidRequestException.class) public void differentKeyColumnFamilies() throws ConfigurationException, InvalidRequestException { CFMetaData metadata = makeCfMetaData("ks1", "cf1", TriggerDefinition.create("test", DifferentKeyTrigger.class.getName())); - TriggerExecutor.instance.execute(bytes("k1"), makeCf(metadata, "v1", null)); + TriggerExecutor.instance.execute(makeCf(metadata, "k1", "v1", null)); } @Test public void noTriggerMutations() throws ConfigurationException, InvalidRequestException { CFMetaData metadata = makeCfMetaData("ks1", "cf1", TriggerDefinition.create("test", NoOpTrigger.class.getName())); - Mutation rm = new Mutation(bytes("k1"), makeCf(metadata, "v1", null)); + Mutation rm = new Mutation(makeCf(metadata, "k1", "v1", null)); assertNull(TriggerExecutor.instance.execute(Collections.singletonList(rm))); } @@ -72,159 +83,179 @@ public class TriggerExecutorTest public void sameKeySameCfRowMutations() throws ConfigurationException, InvalidRequestException { CFMetaData metadata = makeCfMetaData("ks1", "cf1", TriggerDefinition.create("test", SameKeySameCfTrigger.class.getName())); - ColumnFamily cf1 = makeCf(metadata, "k1v1", null); - ColumnFamily cf2 = makeCf(metadata, "k2v1", null); - Mutation rm1 = new Mutation(bytes("k1"), cf1); - Mutation rm2 = new Mutation(bytes("k2"), cf2); + PartitionUpdate cf1 = makeCf(metadata, "k1", "k1v1", null); + PartitionUpdate cf2 = makeCf(metadata, "k2", "k2v1", null); + Mutation rm1 = new Mutation("ks1", cf1.partitionKey()).add(cf1); + Mutation rm2 = new Mutation("ks1", cf2.partitionKey()).add(cf2); List tmutations = new ArrayList<>(TriggerExecutor.instance.execute(Arrays.asList(rm1, rm2))); assertEquals(2, tmutations.size()); Collections.sort(tmutations, new RmComparator()); - List mutatedCFs = new ArrayList<>(tmutations.get(0).getColumnFamilies()); + List mutatedCFs = new ArrayList<>(tmutations.get(0).getPartitionUpdates()); assertEquals(1, mutatedCFs.size()); - assertEquals(bytes("k1v1"), mutatedCFs.get(0).getColumn(getColumnName(metadata, "c1")).value()); - assertEquals(bytes("trigger"), mutatedCFs.get(0).getColumn(getColumnName(metadata, "c2")).value()); + Row row = mutatedCFs.get(0).iterator().next(); + assertEquals(bytes("k1v1"), row.getCell(metadata.getColumnDefinition(bytes("c1"))).value()); + assertEquals(bytes("trigger"), row.getCell(metadata.getColumnDefinition(bytes("c2"))).value()); - mutatedCFs = new ArrayList<>(tmutations.get(1).getColumnFamilies()); + mutatedCFs = new ArrayList<>(tmutations.get(1).getPartitionUpdates()); assertEquals(1, mutatedCFs.size()); - assertEquals(bytes("k2v1"), mutatedCFs.get(0).getColumn(getColumnName(metadata, "c1")).value()); - assertEquals(bytes("trigger"), mutatedCFs.get(0).getColumn(getColumnName(metadata, "c2")).value()); + row = mutatedCFs.get(0).iterator().next(); + assertEquals(bytes("k2v1"), row.getCell(metadata.getColumnDefinition(bytes("c1"))).value()); + assertEquals(bytes("trigger"), row.getCell(metadata.getColumnDefinition(bytes("c2"))).value()); } @Test public void sameKeySameCfPartialRowMutations() throws ConfigurationException, InvalidRequestException { CFMetaData metadata = makeCfMetaData("ks1", "cf1", TriggerDefinition.create("test", SameKeySameCfPartialTrigger.class.getName())); - ColumnFamily cf1 = makeCf(metadata, "k1v1", null); - ColumnFamily cf2 = makeCf(metadata, "k2v1", null); - Mutation rm1 = new Mutation(bytes("k1"), cf1); - Mutation rm2 = new Mutation(bytes("k2"), cf2); + PartitionUpdate cf1 = makeCf(metadata, "k1", "k1v1", null); + PartitionUpdate cf2 = makeCf(metadata, "k2", "k2v1", null); + Mutation rm1 = new Mutation("ks1", cf1.partitionKey()).add(cf1); + Mutation rm2 = new Mutation("ks1", cf2.partitionKey()).add(cf2); List tmutations = new ArrayList<>(TriggerExecutor.instance.execute(Arrays.asList(rm1, rm2))); assertEquals(2, tmutations.size()); Collections.sort(tmutations, new RmComparator()); - List mutatedCFs = new ArrayList<>(tmutations.get(0).getColumnFamilies()); + List mutatedCFs = new ArrayList<>(tmutations.get(0).getPartitionUpdates()); assertEquals(1, mutatedCFs.size()); - assertEquals(bytes("k1v1"), mutatedCFs.get(0).getColumn(getColumnName(metadata, "c1")).value()); - assertNull(mutatedCFs.get(0).getColumn(getColumnName(metadata, "c2"))); + Row row = mutatedCFs.get(0).iterator().next(); + assertEquals(bytes("k1v1"), row.getCell(metadata.getColumnDefinition(bytes("c1"))).value()); + assertNull(row.getCell(metadata.getColumnDefinition(bytes("c2")))); - mutatedCFs = new ArrayList<>(tmutations.get(1).getColumnFamilies()); + mutatedCFs = new ArrayList<>(tmutations.get(1).getPartitionUpdates()); assertEquals(1, mutatedCFs.size()); - assertEquals(bytes("k2v1"), mutatedCFs.get(0).getColumn(getColumnName(metadata, "c1")).value()); - assertEquals(bytes("trigger"), mutatedCFs.get(0).getColumn(getColumnName(metadata, "c2")).value()); + row = mutatedCFs.get(0).iterator().next(); + assertEquals(bytes("k2v1"), row.getCell(metadata.getColumnDefinition(bytes("c1"))).value()); + assertEquals(bytes("trigger"), row.getCell(metadata.getColumnDefinition(bytes("c2"))).value()); } @Test public void sameKeyDifferentCfRowMutations() throws ConfigurationException, InvalidRequestException { CFMetaData metadata = makeCfMetaData("ks1", "cf1", TriggerDefinition.create("test", SameKeyDifferentCfTrigger.class.getName())); - ColumnFamily cf1 = makeCf(metadata, "k1v1", null); - ColumnFamily cf2 = makeCf(metadata, "k2v1", null); - Mutation rm1 = new Mutation(bytes("k1"), cf1); - Mutation rm2 = new Mutation(bytes("k2"), cf2); + PartitionUpdate cf1 = makeCf(metadata, "k1", "k1v1", null); + PartitionUpdate cf2 = makeCf(metadata, "k2", "k2v1", null); + Mutation rm1 = new Mutation("ks1", cf1.partitionKey()).add(cf1); + Mutation rm2 = new Mutation("ks1", cf2.partitionKey()).add(cf2); List tmutations = new ArrayList<>(TriggerExecutor.instance.execute(Arrays.asList(rm1, rm2))); assertEquals(2, tmutations.size()); Collections.sort(tmutations, new RmComparator()); - List mutatedCFs = new ArrayList<>(tmutations.get(0).getColumnFamilies()); + List mutatedCFs = new ArrayList<>(tmutations.get(0).getPartitionUpdates()); + assertEquals(2, mutatedCFs.size()); + for (PartitionUpdate update : mutatedCFs) + { + if (update.metadata().cfName.equals("cf1")) + { + Row row = update.iterator().next(); + assertEquals(bytes("k1v1"), row.getCell(metadata.getColumnDefinition(bytes("c1"))).value()); + assertNull(row.getCell(metadata.getColumnDefinition(bytes("c2")))); + } + else + { + Row row = update.iterator().next(); + assertNull(row.getCell(metadata.getColumnDefinition(bytes("c1")))); + assertEquals(bytes("trigger"), row.getCell(metadata.getColumnDefinition(bytes("c2"))).value()); + } + } + + mutatedCFs = new ArrayList<>(tmutations.get(1).getPartitionUpdates()); assertEquals(2, mutatedCFs.size()); - Collections.sort(mutatedCFs, new CfComparator()); - assertEquals(bytes("k1v1"), mutatedCFs.get(0).getColumn(getColumnName(metadata, "c1")).value()); - assertNull(mutatedCFs.get(0).getColumn(getColumnName(metadata, "c2"))); - assertNull(mutatedCFs.get(1).getColumn(getColumnName(metadata, "c1"))); - assertEquals(bytes("trigger"), mutatedCFs.get(1).getColumn(getColumnName(metadata, "c2")).value()); - - mutatedCFs = new ArrayList<>(tmutations.get(1).getColumnFamilies()); - assertEquals(2, mutatedCFs.size()); - - Collections.sort(mutatedCFs, new CfComparator()); - assertEquals(bytes("k2v1"), mutatedCFs.get(0).getColumn(getColumnName(metadata, "c1")).value()); - assertNull(mutatedCFs.get(0).getColumn(getColumnName(metadata, "c2"))); - assertNull(mutatedCFs.get(1).getColumn(getColumnName(metadata, "c1"))); - assertEquals(bytes("trigger"), mutatedCFs.get(1).getColumn(getColumnName(metadata, "c2")).value()); + for (PartitionUpdate update : mutatedCFs) + { + if (update.metadata().cfName.equals("cf1")) + { + Row row = update.iterator().next(); + assertEquals(bytes("k2v1"), row.getCell(metadata.getColumnDefinition(bytes("c1"))).value()); + assertNull(row.getCell(metadata.getColumnDefinition(bytes("c2")))); + } + else + { + Row row = update.iterator().next(); + assertNull(row.getCell(metadata.getColumnDefinition(bytes("c1")))); + assertEquals(bytes("trigger"), row.getCell(metadata.getColumnDefinition(bytes("c2"))).value()); + } + } } @Test public void sameKeyDifferentKsRowMutations() throws ConfigurationException, InvalidRequestException { CFMetaData metadata = makeCfMetaData("ks1", "cf1", TriggerDefinition.create("test", SameKeyDifferentKsTrigger.class.getName())); - ColumnFamily cf1 = makeCf(metadata, "k1v1", null); - ColumnFamily cf2 = makeCf(metadata, "k2v1", null); - Mutation rm1 = new Mutation(bytes("k1"), cf1); - Mutation rm2 = new Mutation(bytes("k2"), cf2); + PartitionUpdate cf1 = makeCf(metadata, "k1", "k1v1", null); + PartitionUpdate cf2 = makeCf(metadata, "k2", "k2v1", null); + Mutation rm1 = new Mutation("ks1", cf1.partitionKey()).add(cf1); + Mutation rm2 = new Mutation("ks1", cf2.partitionKey()).add(cf2); List tmutations = new ArrayList<>(TriggerExecutor.instance.execute(Arrays.asList(rm1, rm2))); assertEquals(4, tmutations.size()); Collections.sort(tmutations, new RmComparator()); - List mutatedCFs = new ArrayList<>(tmutations.get(0).getColumnFamilies()); + List mutatedCFs = new ArrayList<>(tmutations.get(0).getPartitionUpdates()); assertEquals(1, mutatedCFs.size()); - assertEquals(bytes("k1v1"), mutatedCFs.get(0).getColumn(getColumnName(metadata, "c1")).value()); - assertNull(mutatedCFs.get(0).getColumn(getColumnName(metadata, "c2"))); + Row row = mutatedCFs.get(0).iterator().next(); + assertEquals(bytes("k1v1"), row.getCell(metadata.getColumnDefinition(bytes("c1"))).value()); + assertNull(row.getCell(metadata.getColumnDefinition(bytes("c2")))); - mutatedCFs = new ArrayList<>(tmutations.get(1).getColumnFamilies()); + mutatedCFs = new ArrayList<>(tmutations.get(1).getPartitionUpdates()); assertEquals(1, mutatedCFs.size()); - assertEquals(bytes("k2v1"), mutatedCFs.get(0).getColumn(getColumnName(metadata, "c1")).value()); - assertNull(mutatedCFs.get(0).getColumn(getColumnName(metadata, "c2"))); + row = mutatedCFs.get(0).iterator().next(); + assertEquals(bytes("k2v1"), row.getCell(metadata.getColumnDefinition(bytes("c1"))).value()); + assertNull(row.getCell(metadata.getColumnDefinition(bytes("c2")))); - mutatedCFs = new ArrayList<>(tmutations.get(2).getColumnFamilies()); + mutatedCFs = new ArrayList<>(tmutations.get(2).getPartitionUpdates()); assertEquals(1, mutatedCFs.size()); - assertNull(mutatedCFs.get(0).getColumn(getColumnName(metadata, "c1"))); - assertEquals(bytes("trigger"), mutatedCFs.get(0).getColumn(getColumnName(metadata, "c2")).value()); + row = mutatedCFs.get(0).iterator().next(); + assertNull(row.getCell(metadata.getColumnDefinition(bytes("c1")))); + assertEquals(bytes("trigger"), row.getCell(metadata.getColumnDefinition(bytes("c2"))).value()); - mutatedCFs = new ArrayList<>(tmutations.get(3).getColumnFamilies()); + mutatedCFs = new ArrayList<>(tmutations.get(3).getPartitionUpdates()); assertEquals(1, mutatedCFs.size()); - assertNull(mutatedCFs.get(0).getColumn(getColumnName(metadata, "c1"))); - assertEquals(bytes("trigger"), mutatedCFs.get(0).getColumn(getColumnName(metadata, "c2")).value()); + row = mutatedCFs.get(0).iterator().next(); + assertNull(row.getCell(metadata.getColumnDefinition(bytes("c1")))); + assertEquals(bytes("trigger"), row.getCell(metadata.getColumnDefinition(bytes("c2"))).value()); } @Test public void differentKeyRowMutations() throws ConfigurationException, InvalidRequestException { + CFMetaData metadata = makeCfMetaData("ks1", "cf1", TriggerDefinition.create("test", DifferentKeyTrigger.class.getName())); - ColumnFamily cf = makeCf(metadata, "v1", null); - Mutation rm = new Mutation(UTF8Type.instance.fromString("k1"), cf); + PartitionUpdate cf1 = makeCf(metadata, "k1", "v1", null); + Mutation rm = new Mutation("ks1", cf1.partitionKey()).add(cf1); List tmutations = new ArrayList<>(TriggerExecutor.instance.execute(Arrays.asList(rm))); assertEquals(2, tmutations.size()); Collections.sort(tmutations, new RmComparator()); - assertEquals(bytes("k1"), tmutations.get(0).key()); - assertEquals(bytes("otherKey"), tmutations.get(1).key()); + assertEquals(bytes("k1"), tmutations.get(0).key().getKey()); + assertEquals(bytes("otherKey"), tmutations.get(1).key().getKey()); - List mutatedCFs = new ArrayList<>(tmutations.get(0).getColumnFamilies()); + List mutatedCFs = new ArrayList<>(tmutations.get(0).getPartitionUpdates()); assertEquals(1, mutatedCFs.size()); - assertEquals(bytes("v1"), mutatedCFs.get(0).getColumn(getColumnName(metadata, "c1")).value()); - assertNull(mutatedCFs.get(0).getColumn(getColumnName(metadata, "c2"))); + Row row = mutatedCFs.get(0).iterator().next(); + assertEquals(bytes("v1"), row.getCell(metadata.getColumnDefinition(bytes("c1"))).value()); + assertNull(row.getCell(metadata.getColumnDefinition(bytes("c2")))); - mutatedCFs = new ArrayList<>(tmutations.get(1).getColumnFamilies()); + mutatedCFs = new ArrayList<>(tmutations.get(1).getPartitionUpdates()); assertEquals(1, mutatedCFs.size()); - assertNull(mutatedCFs.get(0).getColumn(getColumnName(metadata, "c1"))); - assertEquals(bytes("trigger"), mutatedCFs.get(0).getColumn(getColumnName(metadata, "c2")).value()); + row = mutatedCFs.get(0).iterator().next(); + assertEquals(bytes("trigger"), row.getCell(metadata.getColumnDefinition(bytes("c2"))).value()); + assertNull(row.getCell(metadata.getColumnDefinition(bytes("c1")))); } private static CFMetaData makeCfMetaData(String ks, String cf, TriggerDefinition trigger) { + CFMetaData metadata = CFMetaData.Builder.create(ks, cf) + .addPartitionKey("pkey", UTF8Type.instance) + .addRegularColumn("c1", UTF8Type.instance) + .addRegularColumn("c2", UTF8Type.instance) + .build(); - CFMetaData metadata = CFMetaData.sparseCFMetaData(ks, cf, CompositeType.getInstance(UTF8Type.instance)); - - metadata.keyValidator(UTF8Type.instance); - metadata.addOrReplaceColumnDefinition(ColumnDefinition.partitionKeyDef(metadata, - UTF8Type.instance.fromString("pkey"), - UTF8Type.instance, - null)); - metadata.addOrReplaceColumnDefinition(ColumnDefinition.regularDef(metadata, - UTF8Type.instance.fromString("c1"), - UTF8Type.instance, - 0)); - metadata.addOrReplaceColumnDefinition(ColumnDefinition.regularDef(metadata, - UTF8Type.instance.fromString("c2"), - UTF8Type.instance, - 0)); try { if (trigger != null) @@ -235,30 +266,32 @@ public class TriggerExecutorTest throw new AssertionError(e); } - return metadata.rebuild(); + return metadata; } - private static ColumnFamily makeCf(CFMetaData metadata, String columnValue1, String columnValue2) + private static PartitionUpdate makeCf(CFMetaData metadata, String key, String columnValue1, String columnValue2) { - ColumnFamily cf = ArrayBackedSortedColumns.factory.create(metadata); + PartitionUpdate update = new PartitionUpdate(metadata, Util.dk(key), metadata.partitionColumns(), 1); + + LivenessInfo info = SimpleLivenessInfo.forUpdate(FBUtilities.timestampMicros(), LivenessInfo.NO_TTL, FBUtilities.nowInSeconds(), metadata); if (columnValue1 != null) - cf.addColumn(new BufferCell(getColumnName(metadata, "c1"), bytes(columnValue1))); - + { + update.writer().writeCell(metadata.getColumnDefinition(bytes("c1")), false, bytes(columnValue1), info, null); + update.writer().endOfRow(); + } if (columnValue2 != null) - cf.addColumn(new BufferCell(getColumnName(metadata, "c2"), bytes(columnValue2))); + { + update.writer().writeCell(metadata.getColumnDefinition(bytes("c2")), false, bytes(columnValue1), info, null); + update.writer().endOfRow(); + } - return cf; - } - - private static CellName getColumnName(CFMetaData metadata, String stringName) - { - return metadata.comparator.makeCellName(stringName); + return update; } public static class NoOpTrigger implements ITrigger { - public Collection augment(ByteBuffer key, ColumnFamily update) + public Collection augment(Partition partition) { return null; } @@ -266,54 +299,54 @@ public class TriggerExecutorTest public static class SameKeySameCfTrigger implements ITrigger { - public Collection augment(ByteBuffer key, ColumnFamily update) + public Collection augment(Partition partition) { - ColumnFamily cf = ArrayBackedSortedColumns.factory.create(update.metadata()); - cf.addColumn(new BufferCell(getColumnName(update.metadata(), "c2"), bytes("trigger"))); - return Collections.singletonList(new Mutation(update.metadata().ksName, key, cf)); + RowUpdateBuilder builder = new RowUpdateBuilder(partition.metadata(), FBUtilities.timestampMicros(), partition.partitionKey().getKey()); + builder.add("c2", bytes("trigger")); + return Collections.singletonList(builder.build()); } } public static class SameKeySameCfPartialTrigger implements ITrigger { - public Collection augment(ByteBuffer key, ColumnFamily update) + public Collection augment(Partition partition) { - if (!key.equals(bytes("k2"))) + if (!partition.partitionKey().getKey().equals(bytes("k2"))) return null; - ColumnFamily cf = ArrayBackedSortedColumns.factory.create(update.metadata()); - cf.addColumn(new BufferCell(getColumnName(update.metadata(), "c2"), bytes("trigger"))); - return Collections.singletonList(new Mutation(update.metadata().ksName, key, cf)); + RowUpdateBuilder builder = new RowUpdateBuilder(partition.metadata(), FBUtilities.timestampMicros(), partition.partitionKey().getKey()); + builder.add("c2", bytes("trigger")); + return Collections.singletonList(builder.build()); } } public static class SameKeyDifferentCfTrigger implements ITrigger { - public Collection augment(ByteBuffer key, ColumnFamily update) + public Collection augment(Partition partition) { - ColumnFamily cf = ArrayBackedSortedColumns.factory.create(makeCfMetaData(update.metadata().ksName, "otherCf", null)); - cf.addColumn(new BufferCell(getColumnName(update.metadata(), "c2"), bytes("trigger"))); - return Collections.singletonList(new Mutation(cf.metadata().ksName, key, cf)); + RowUpdateBuilder builder = new RowUpdateBuilder(makeCfMetaData(partition.metadata().ksName, "otherCf", null), FBUtilities.timestampMicros(), partition.partitionKey().getKey()); + builder.add("c2", bytes("trigger")); + return Collections.singletonList(builder.build()); } } public static class SameKeyDifferentKsTrigger implements ITrigger { - public Collection augment(ByteBuffer key, ColumnFamily update) + public Collection augment(Partition partition) { - ColumnFamily cf = ArrayBackedSortedColumns.factory.create(makeCfMetaData("otherKs", "otherCf", null)); - cf.addColumn(new BufferCell(getColumnName(update.metadata(), "c2"), bytes("trigger"))); - return Collections.singletonList(new Mutation(cf.metadata().ksName, key, cf)); + RowUpdateBuilder builder = new RowUpdateBuilder(makeCfMetaData("otherKs", "otherCf", null), FBUtilities.timestampMicros(), partition.partitionKey().getKey()); + builder.add("c2", bytes("trigger")); + return Collections.singletonList(builder.build()); } } public static class DifferentKeyTrigger implements ITrigger { - public Collection augment(ByteBuffer key, ColumnFamily update) + public Collection augment(Partition partition) { - ColumnFamily cf = ArrayBackedSortedColumns.factory.create(update.metadata()); - cf.addColumn(new BufferCell(getColumnName(update.metadata(), "c2"), bytes("trigger"))); - return Collections.singletonList(new Mutation(cf.metadata().ksName, bytes("otherKey"), cf)); + RowUpdateBuilder builder = new RowUpdateBuilder(makeCfMetaData("otherKs", "otherCf", null), FBUtilities.timestampMicros(), "otherKey"); + builder.add("c2", bytes("trigger")); + return Collections.singletonList(builder.build()); } } @@ -326,9 +359,9 @@ public class TriggerExecutorTest } } - private static class CfComparator implements Comparator + private static class CfComparator implements Comparator { - public int compare(ColumnFamily cf1, ColumnFamily cf2) + public int compare(Partition cf1, Partition cf2) { return cf1.metadata().cfName.compareTo(cf2.metadata().cfName); } diff --git a/test/unit/org/apache/cassandra/triggers/TriggersSchemaTest.java b/test/unit/org/apache/cassandra/triggers/TriggersSchemaTest.java index 577e7d3e75..7097b9aef6 100644 --- a/test/unit/org/apache/cassandra/triggers/TriggersSchemaTest.java +++ b/test/unit/org/apache/cassandra/triggers/TriggersSchemaTest.java @@ -31,9 +31,7 @@ import org.apache.cassandra.exceptions.ConfigurationException; import org.apache.cassandra.locator.SimpleStrategy; import org.apache.cassandra.service.MigrationManager; -import static org.junit.Assert.assertEquals; -import static org.junit.Assert.assertFalse; -import static org.junit.Assert.assertTrue; +import static org.junit.Assert.*; public class TriggersSchemaTest { @@ -55,10 +53,10 @@ public class TriggersSchemaTest CFMetaData cfm1 = CFMetaData.compile(String.format("CREATE TABLE %s (k int PRIMARY KEY, v int)", cfName), ksName); cfm1.addTriggerDefinition(td); KSMetaData ksm = KSMetaData.newKeyspace(ksName, - SimpleStrategy.class, - Collections.singletonMap("replication_factor", "1"), - true, - Collections.singletonList(cfm1)); + SimpleStrategy.class, + Collections.singletonMap("replication_factor", "1"), + true, + Collections.singletonList(cfm1)); MigrationManager.announceNewKeyspace(ksm); CFMetaData cfm2 = Schema.instance.getCFMetaData(ksName, cfName); @@ -71,10 +69,10 @@ public class TriggersSchemaTest public void addNewCfWithTriggerToKs() throws Exception { KSMetaData ksm = KSMetaData.newKeyspace(ksName, - SimpleStrategy.class, - Collections.singletonMap("replication_factor", "1"), - true, - Collections.EMPTY_LIST); + SimpleStrategy.class, + Collections.singletonMap("replication_factor", "1"), + true, + Collections.EMPTY_LIST); MigrationManager.announceNewKeyspace(ksm); CFMetaData cfm1 = CFMetaData.compile(String.format("CREATE TABLE %s (k int PRIMARY KEY, v int)", cfName), ksName); @@ -94,10 +92,10 @@ public class TriggersSchemaTest { CFMetaData cfm1 = CFMetaData.compile(String.format("CREATE TABLE %s (k int PRIMARY KEY, v int)", cfName), ksName); KSMetaData ksm = KSMetaData.newKeyspace(ksName, - SimpleStrategy.class, - Collections.singletonMap("replication_factor", "1"), - true, - Collections.singletonList(cfm1)); + SimpleStrategy.class, + Collections.singletonMap("replication_factor", "1"), + true, + Collections.singletonList(cfm1)); MigrationManager.announceNewKeyspace(ksm); CFMetaData cfm2 = Schema.instance.getCFMetaData(ksName, cfName).copy(); @@ -118,10 +116,10 @@ public class TriggersSchemaTest CFMetaData cfm1 = CFMetaData.compile(String.format("CREATE TABLE %s (k int PRIMARY KEY, v int)", cfName), ksName); cfm1.addTriggerDefinition(td); KSMetaData ksm = KSMetaData.newKeyspace(ksName, - SimpleStrategy.class, - Collections.singletonMap("replication_factor", "1"), - true, - Collections.singletonList(cfm1)); + SimpleStrategy.class, + Collections.singletonMap("replication_factor", "1"), + true, + Collections.singletonList(cfm1)); MigrationManager.announceNewKeyspace(ksm); CFMetaData cfm2 = Schema.instance.getCFMetaData(ksName, cfName).copy(); diff --git a/test/unit/org/apache/cassandra/triggers/TriggersTest.java b/test/unit/org/apache/cassandra/triggers/TriggersTest.java index b0a5aca6fc..13ecbe92d7 100644 --- a/test/unit/org/apache/cassandra/triggers/TriggersTest.java +++ b/test/unit/org/apache/cassandra/triggers/TriggersTest.java @@ -22,7 +22,10 @@ import java.nio.ByteBuffer; import java.util.Collection; import java.util.Collections; -import org.junit.*; +import org.junit.AfterClass; +import org.junit.Before; +import org.junit.BeforeClass; +import org.junit.Test; import org.apache.cassandra.SchemaLoader; import org.apache.cassandra.config.Schema; @@ -31,10 +34,13 @@ import org.apache.cassandra.cql3.UntypedResultSet; import org.apache.cassandra.db.*; import org.apache.cassandra.db.ConsistencyLevel; import org.apache.cassandra.db.Mutation; +import org.apache.cassandra.db.partitions.Partition; +import org.apache.cassandra.db.partitions.PartitionUpdate; import org.apache.cassandra.exceptions.ConfigurationException; import org.apache.cassandra.exceptions.RequestExecutionException; import org.apache.cassandra.service.StorageService; import org.apache.cassandra.thrift.*; +import org.apache.cassandra.utils.FBUtilities; import org.apache.thrift.protocol.TBinaryProtocol; import static org.apache.cassandra.utils.ByteBufferUtil.bytes; @@ -55,12 +61,12 @@ public class TriggersTest public static void beforeTest() throws ConfigurationException { SchemaLoader.loadSchema(); - StorageService.instance.initServer(0); } @Before public void setup() throws Exception { + StorageService.instance.initServer(0); if (thriftServer == null || ! thriftServer.isRunning()) { thriftServer = new ThriftServer(InetAddress.getLocalHost(), 9170, 50); @@ -277,29 +283,6 @@ public class TriggersTest } } - @Test(expected=RuntimeException.class) - public void ifTriggerThrowsErrorNoMutationsAreApplied() throws Exception - { - String cf = "cf" + System.nanoTime(); - try - { - setupTableWithTrigger(cf, ErrorTrigger.class); - String cql = String.format("INSERT INTO %s.%s (k, v1) VALUES (11, 11)", ksName, cf); - QueryProcessor.process(cql, ConsistencyLevel.ONE); - } - catch (Exception e) - { - Throwable cause = e.getCause(); - assertTrue((cause instanceof org.apache.cassandra.exceptions.InvalidRequestException)); - assertTrue(cause.getMessage().equals(ErrorTrigger.MESSAGE)); - throw e; - } - finally - { - assertUpdateNotExecuted(cf, 11); - } - } - private void setupTableWithTrigger(String cf, Class triggerImpl) throws RequestExecutionException { @@ -315,7 +298,7 @@ public class TriggersTest private void assertUpdateIsAugmented(int key) { UntypedResultSet rs = QueryProcessor.executeInternal( - String.format("SELECT * FROM %s.%s WHERE k=%s", ksName, cfName, key)); + String.format("SELECT * FROM %s.%s WHERE k=%s", ksName, cfName, key)); assertTrue(String.format("Expected value (%s) for augmented cell v2 was not found", key), rs.one().has("v2")); assertEquals(999, rs.one().getInt("v2")); } @@ -330,7 +313,7 @@ public class TriggersTest private org.apache.cassandra.thrift.Column getColumnForInsert(String columnName, int value) { org.apache.cassandra.thrift.Column column = new org.apache.cassandra.thrift.Column(); - column.setName(Schema.instance.getCFMetaData(ksName, cfName).comparator.asAbstractType().fromString(columnName)); + column.setName(LegacyLayout.makeLegacyComparator(Schema.instance.getCFMetaData(ksName, cfName)).fromString(columnName)); column.setValue(bytes(value)); column.setTimestamp(System.currentTimeMillis()); return column; @@ -338,43 +321,35 @@ public class TriggersTest public static class TestTrigger implements ITrigger { - public Collection augment(ByteBuffer key, ColumnFamily update) + public Collection augment(Partition partition) { - ColumnFamily extraUpdate = update.cloneMeShallow(ArrayBackedSortedColumns.factory, false); - extraUpdate.addColumn(new BufferCell(update.metadata().comparator.makeCellName(bytes("v2")), bytes(999))); - return Collections.singletonList(new Mutation(ksName, key, extraUpdate)); + RowUpdateBuilder update = new RowUpdateBuilder(partition.metadata(), FBUtilities.timestampMicros(), partition.partitionKey().getKey()); + update.add("v2", 999); + + return Collections.singletonList(update.build()); } } public static class CrossPartitionTrigger implements ITrigger { - public Collection augment(ByteBuffer key, ColumnFamily update) + public Collection augment(Partition partition) { - ColumnFamily extraUpdate = update.cloneMeShallow(ArrayBackedSortedColumns.factory, false); - extraUpdate.addColumn(new BufferCell(update.metadata().comparator.makeCellName(bytes("v2")), bytes(999))); + RowUpdateBuilder update = new RowUpdateBuilder(partition.metadata(), FBUtilities.timestampMicros(), toInt(partition.partitionKey().getKey()) + 1000); + update.add("v2", 999); - int newKey = toInt(key) + 1000; - return Collections.singletonList(new Mutation(ksName, bytes(newKey), extraUpdate)); + return Collections.singletonList(update.build()); } } public static class CrossTableTrigger implements ITrigger { - public Collection augment(ByteBuffer key, ColumnFamily update) + public Collection augment(Partition partition) { - ColumnFamily extraUpdate = ArrayBackedSortedColumns.factory.create(ksName, otherCf); - extraUpdate.addColumn(new BufferCell(extraUpdate.metadata().comparator.makeCellName(bytes("v2")), bytes(999))); - return Collections.singletonList(new Mutation(ksName, key, extraUpdate)); + + RowUpdateBuilder update = new RowUpdateBuilder(Schema.instance.getCFMetaData(ksName, otherCf), FBUtilities.timestampMicros(), partition.partitionKey().getKey()); + update.add("v2", 999); + + return Collections.singletonList(update.build()); } } - - public static class ErrorTrigger implements ITrigger - { - public static final String MESSAGE = "Thrown by ErrorTrigger"; - public Collection augment(ByteBuffer partitionKey, ColumnFamily update) - { - throw new org.apache.cassandra.exceptions.InvalidRequestException(MESSAGE); - } - } - } diff --git a/test/unit/org/apache/cassandra/utils/BTreeTest.java b/test/unit/org/apache/cassandra/utils/BTreeTest.java index e1bf388ecb..6e1786d0bb 100644 --- a/test/unit/org/apache/cassandra/utils/BTreeTest.java +++ b/test/unit/org/apache/cassandra/utils/BTreeTest.java @@ -23,12 +23,9 @@ import java.util.concurrent.ThreadLocalRandom; import org.junit.Test; import org.apache.cassandra.utils.btree.BTree; -import org.apache.cassandra.utils.btree.BTreeSet; import org.apache.cassandra.utils.btree.UpdateFunction; -import static org.junit.Assert.assertEquals; -import static org.junit.Assert.assertArrayEquals; -import static org.junit.Assert.assertTrue; +import static org.junit.Assert.*; public class BTreeTest { @@ -40,7 +37,7 @@ public class BTreeTest ints[i] = new Integer(i); } - static final UpdateFunction updateF = new UpdateFunction() + static final UpdateFunction updateF = new UpdateFunction() { public Integer apply(Integer replacing, Integer update) { @@ -63,6 +60,28 @@ public class BTreeTest } }; + private static final UpdateFunction noOp = new UpdateFunction() + { + public Integer apply(Integer replacing, Integer update) + { + return update; + } + + public boolean abortEarly() + { + return false; + } + + public void allocated(long heapSize) + { + } + + public Integer apply(Integer k) + { + return k; + } + }; + private static List seq(int count) { List r = new ArrayList<>(); @@ -97,20 +116,14 @@ public class BTreeTest public void testBuilding_UpdateFunctionReplacement() { for (int i = 0; i < 20 ; i++) - { - checkResult(i, BTree.build(seq(i), CMP, true, updateF)); - checkResult(i, BTree.build(rand(i), CMP, false, updateF)); - } + checkResult(i, BTree.build(seq(i), updateF)); } @Test public void testUpdate_UpdateFunctionReplacement() { for (int i = 0; i < 20 ; i++) - { - checkResult(i, BTree.update(BTree.build(seq(i), CMP, true, UpdateFunction.NoOp.instance()), CMP, seq(i), true, updateF)); - checkResult(i, BTree.update(BTree.build(rand(i), CMP, false, UpdateFunction.NoOp.instance()), CMP, rand(i), false, updateF)); - } + checkResult(i, BTree.update(BTree.build(seq(i), noOp), CMP, seq(i), updateF)); } /** @@ -123,31 +136,31 @@ public class BTreeTest Object[] btree = new Object[0]; CallsMonitor monitor = new CallsMonitor(); - btree = BTree.update(btree, CMP, Arrays.asList(1), true, monitor); + btree = BTree.update(btree, CMP, Arrays.asList(1), monitor); assertArrayEquals(new Object[] {1, null}, btree); assertEquals(1, monitor.getNumberOfCalls(1)); monitor.clear(); - btree = BTree.update(btree, CMP, Arrays.asList(2), true, monitor); + btree = BTree.update(btree, CMP, Arrays.asList(2), monitor); assertArrayEquals(new Object[] {1, 2}, btree); assertEquals(1, monitor.getNumberOfCalls(2)); // with existing value monitor.clear(); - btree = BTree.update(btree, CMP, Arrays.asList(1), true, monitor); + btree = BTree.update(btree, CMP, Arrays.asList(1), monitor); assertArrayEquals(new Object[] {1, 2}, btree); assertEquals(1, monitor.getNumberOfCalls(1)); // with two non-existing values monitor.clear(); - btree = BTree.update(btree, CMP, Arrays.asList(3, 4), true, monitor); + btree = BTree.update(btree, CMP, Arrays.asList(3, 4), monitor); assertArrayEquals(new Object[] {1, 2, 3, 4}, btree); assertEquals(1, monitor.getNumberOfCalls(3)); assertEquals(1, monitor.getNumberOfCalls(4)); - // with one existing value and one non existing value in disorder + // with one existing value and one non existing value monitor.clear(); - btree = BTree.update(btree, CMP, Arrays.asList(5, 2), false, monitor); + btree = BTree.update(btree, CMP, Arrays.asList(2, 5), monitor); assertArrayEquals(new Object[] {3, new Object[]{1, 2}, new Object[]{4, 5}}, btree); assertEquals(1, monitor.getNumberOfCalls(2)); assertEquals(1, monitor.getNumberOfCalls(5)); @@ -160,18 +173,18 @@ public class BTreeTest public void testBuilding_UpdateFunctionCallBack() { CallsMonitor monitor = new CallsMonitor(); - Object[] btree = BTree.build(Arrays.asList(1), CMP, true, monitor); + Object[] btree = BTree.build(Arrays.asList(1), monitor); assertArrayEquals(new Object[] {1, null}, btree); assertEquals(1, monitor.getNumberOfCalls(1)); monitor.clear(); - btree = BTree.build(Arrays.asList(1, 2), CMP, true, monitor); + btree = BTree.build(Arrays.asList(1, 2), monitor); assertArrayEquals(new Object[] {1, 2}, btree); assertEquals(1, monitor.getNumberOfCalls(1)); assertEquals(1, monitor.getNumberOfCalls(2)); monitor.clear(); - btree = BTree.build(Arrays.asList(3, 1, 2), CMP, false, monitor); + btree = BTree.build(Arrays.asList(1, 2, 3), monitor); assertArrayEquals(new Object[] {1, 2, 3, null}, btree); assertEquals(1, monitor.getNumberOfCalls(1)); assertEquals(1, monitor.getNumberOfCalls(2)); @@ -180,31 +193,27 @@ public class BTreeTest private static void checkResult(int count, Object[] btree) { - BTreeSet vs = new BTreeSet<>(btree, CMP); - assert vs.size() == count; + Iterator iter = BTree.slice(btree, true); int i = 0; - for (Integer j : vs) - assertEquals(j, ints[i++]); + while (iter.hasNext()) + assertEquals(iter.next(), ints[i++]); + assertFalse(iter.hasNext()); } @Test public void testClearOnAbort() { - final Comparator cmp = new Comparator() - { - public int compare(String o1, String o2) - { - return o1.compareTo(o2); - } - }; + Object[] btree = BTree.build(seq(2), noOp); + Object[] copy = Arrays.copyOf(btree, btree.length); + BTree.update(btree, CMP, seq(94), new AbortAfterX(90)); - Object[] btree = BTree.build(ranges(range(0, 8)), cmp, true, UpdateFunction.NoOp.instance()); - BTree.update(btree, cmp, ranges(range(0, 94)), false, new AbortAfterX(90)); - btree = BTree.update(btree, cmp, ranges(range(0, 94)), false, UpdateFunction.NoOp.instance()); - assertTrue(BTree.isWellFormed(btree, cmp)); + assertArrayEquals(copy, btree); + + btree = BTree.update(btree, CMP, seq(94), noOp); + assertTrue(BTree.isWellFormed(btree, CMP)); } - private static final class AbortAfterX implements UpdateFunction + private static final class AbortAfterX implements UpdateFunction { int counter; final int abortAfter; @@ -212,7 +221,7 @@ public class BTreeTest { this.abortAfter = abortAfter; } - public String apply(String replacing, String update) + public Integer apply(Integer replacing, Integer update) { return update; } @@ -223,33 +232,16 @@ public class BTreeTest public void allocated(long heapSize) { } - public String apply(String v) + public Integer apply(Integer v) { return v; } } - private static int[] range(int lb, int ub) - { - return new int[] { lb, ub }; - } - - private static List ranges(int[] ... ranges) - { - - List r = new ArrayList<>(); - for (int[] range : ranges) - { - for (int i = range[0] ; i < range[1] ; i+=1) - r.add(Integer.toString(i)); - } - return r; - } - /** * UpdateFunction that count the number of call made to apply for each value. */ - public static final class CallsMonitor implements UpdateFunction + public static final class CallsMonitor implements UpdateFunction { private int[] numberOfCalls = new int[20]; diff --git a/test/unit/org/apache/cassandra/utils/BitSetTest.java b/test/unit/org/apache/cassandra/utils/BitSetTest.java index 9d82edf0c1..a7a0660110 100644 --- a/test/unit/org/apache/cassandra/utils/BitSetTest.java +++ b/test/unit/org/apache/cassandra/utils/BitSetTest.java @@ -24,9 +24,9 @@ import java.util.List; import java.util.Random; import com.google.common.collect.Lists; - -import org.junit.Test; import org.junit.Assert; +import org.junit.Test; + import org.apache.cassandra.io.util.DataOutputBuffer; import org.apache.cassandra.utils.IFilter.FilterKey; import org.apache.cassandra.utils.KeyGenerator.RandomStringGenerator; @@ -79,7 +79,7 @@ public class BitSetTest @Test public void testOffHeapCompatibility() throws IOException { - try (OpenBitSet bs = new OpenBitSet(100000)) + try (OpenBitSet bs = new OpenBitSet(100000)) { populateAndReserialize(bs); } diff --git a/test/unit/org/apache/cassandra/utils/BloomFilterTest.java b/test/unit/org/apache/cassandra/utils/BloomFilterTest.java index 0c8aec636f..72f2825569 100644 --- a/test/unit/org/apache/cassandra/utils/BloomFilterTest.java +++ b/test/unit/org/apache/cassandra/utils/BloomFilterTest.java @@ -18,12 +18,7 @@ */ package org.apache.cassandra.utils; -import java.io.ByteArrayInputStream; -import java.io.DataInputStream; -import java.io.File; -import java.io.FileInputStream; -import java.io.FileOutputStream; -import java.io.IOException; +import java.io.*; import java.nio.ByteBuffer; import java.util.HashSet; import java.util.Iterator; @@ -31,12 +26,13 @@ import java.util.Random; import java.util.Set; import org.junit.*; + import org.apache.cassandra.dht.IPartitioner; import org.apache.cassandra.dht.Murmur3Partitioner; +import org.apache.cassandra.io.util.BufferedDataOutputStreamPlus; import org.apache.cassandra.io.util.DataOutputBuffer; import org.apache.cassandra.io.util.DataOutputStreamPlus; import org.apache.cassandra.io.util.FileUtils; -import org.apache.cassandra.io.util.BufferedDataOutputStreamPlus; import org.apache.cassandra.utils.IFilter.FilterKey; import org.apache.cassandra.utils.KeyGenerator.RandomStringGenerator; @@ -166,27 +162,28 @@ public class BloomFilterTest @Test public void compareCachedKey() { - BloomFilter bf1 = (BloomFilter) FilterFactory.getFilter(FilterTestHelper.ELEMENTS / 2, FilterTestHelper.MAX_FAILURE_RATE, false); - BloomFilter bf2 = (BloomFilter) FilterFactory.getFilter(FilterTestHelper.ELEMENTS / 2, FilterTestHelper.MAX_FAILURE_RATE, false); - BloomFilter bf3 = (BloomFilter) FilterFactory.getFilter(FilterTestHelper.ELEMENTS / 2, FilterTestHelper.MAX_FAILURE_RATE, false); - - RandomStringGenerator gen1 = new KeyGenerator.RandomStringGenerator(new Random().nextInt(), FilterTestHelper.ELEMENTS); - - // make sure all bitsets are empty. - BitSetTest.compare(bf1.bitset, bf2.bitset); - BitSetTest.compare(bf1.bitset, bf3.bitset); - - while (gen1.hasNext()) + try (BloomFilter bf1 = (BloomFilter) FilterFactory.getFilter(FilterTestHelper.ELEMENTS / 2, FilterTestHelper.MAX_FAILURE_RATE, false); + BloomFilter bf2 = (BloomFilter) FilterFactory.getFilter(FilterTestHelper.ELEMENTS / 2, FilterTestHelper.MAX_FAILURE_RATE, false); + BloomFilter bf3 = (BloomFilter) FilterFactory.getFilter(FilterTestHelper.ELEMENTS / 2, FilterTestHelper.MAX_FAILURE_RATE, false)) { - ByteBuffer key = gen1.next(); - FilterKey cached = FilterTestHelper.wrapCached(key); - bf1.add(FilterTestHelper.wrap(key)); - bf2.add(cached); - bf3.add(cached); - } + RandomStringGenerator gen1 = new KeyGenerator.RandomStringGenerator(new Random().nextInt(), FilterTestHelper.ELEMENTS); - BitSetTest.compare(bf1.bitset, bf2.bitset); - BitSetTest.compare(bf1.bitset, bf3.bitset); + // make sure all bitsets are empty. + BitSetTest.compare(bf1.bitset, bf2.bitset); + BitSetTest.compare(bf1.bitset, bf3.bitset); + + while (gen1.hasNext()) + { + ByteBuffer key = gen1.next(); + FilterKey cached = FilterTestHelper.wrapCached(key); + bf1.add(FilterTestHelper.wrap(key)); + bf2.add(cached); + bf3.add(cached); + } + + BitSetTest.compare(bf1.bitset, bf2.bitset); + BitSetTest.compare(bf1.bitset, bf3.bitset); + } } @Test @@ -196,7 +193,7 @@ public class BloomFilterTest ByteBuffer test = ByteBuffer.wrap(new byte[] {0, 1}); File file = FileUtils.createTempFile("bloomFilterTest-", ".dat"); - BloomFilter filter = (BloomFilter) FilterFactory.getFilter(((long)Integer.MAX_VALUE / 8) + 1, 0.01d, true); + BloomFilter filter = (BloomFilter) FilterFactory.getFilter(((long) Integer.MAX_VALUE / 8) + 1, 0.01d, true); filter.add(FilterTestHelper.wrap(test)); DataOutputStreamPlus out = new BufferedDataOutputStreamPlus(new FileOutputStream(file)); FilterFactory.serialize(filter, out); @@ -208,6 +205,7 @@ public class BloomFilterTest BloomFilter filter2 = (BloomFilter) FilterFactory.deserialize(in, true); Assert.assertTrue(filter2.isPresent(FilterTestHelper.wrap(test))); FileUtils.closeQuietly(in); + filter2.close(); } @Test diff --git a/test/unit/org/apache/cassandra/utils/ByteBufferUtilTest.java b/test/unit/org/apache/cassandra/utils/ByteBufferUtilTest.java index 2cbac92b71..3f341022d3 100644 --- a/test/unit/org/apache/cassandra/utils/ByteBufferUtilTest.java +++ b/test/unit/org/apache/cassandra/utils/ByteBufferUtilTest.java @@ -18,12 +18,9 @@ package org.apache.cassandra.utils; -import static org.junit.Assert.assertArrayEquals; -import static org.junit.Assert.assertEquals; - -import java.io.IOException; -import java.io.DataInputStream; import java.io.ByteArrayInputStream; +import java.io.DataInputStream; +import java.io.IOException; import java.nio.ByteBuffer; import java.nio.charset.CharacterCodingException; import java.util.Arrays; @@ -32,6 +29,9 @@ import org.junit.Test; import org.apache.cassandra.io.util.DataOutputBuffer; +import static org.junit.Assert.assertArrayEquals; +import static org.junit.Assert.assertEquals; + public class ByteBufferUtilTest { private static final String s = "cassandra"; @@ -101,11 +101,11 @@ public class ByteBufferUtilTest private void checkLastIndexOf(ByteBuffer bb) { - assert bb.position() + 8 == ByteBufferUtil.lastIndexOf(bb, (byte)'a', bb.position() + 8); - assert bb.position() + 4 == ByteBufferUtil.lastIndexOf(bb, (byte)'a', bb.position() + 7); - assert bb.position() + 3 == ByteBufferUtil.lastIndexOf(bb, (byte)'s', bb.position() + 8); - assert -1 == ByteBufferUtil.lastIndexOf(bb, (byte)'o', bb.position() + 8); - assert -1 == ByteBufferUtil.lastIndexOf(bb, (byte)'d', bb.position() + 5); + assert bb.position() + 8 == ByteBufferUtil.lastIndexOf(bb, (byte) 'a', bb.position() + 8); + assert bb.position() + 4 == ByteBufferUtil.lastIndexOf(bb, (byte) 'a', bb.position() + 7); + assert bb.position() + 3 == ByteBufferUtil.lastIndexOf(bb, (byte) 's', bb.position() + 8); + assert -1 == ByteBufferUtil.lastIndexOf(bb, (byte) 'o', bb.position() + 8); + assert -1 == ByteBufferUtil.lastIndexOf(bb, (byte) 'd', bb.position() + 5); } @Test diff --git a/test/unit/org/apache/cassandra/utils/BytesReadTrackerTest.java b/test/unit/org/apache/cassandra/utils/BytesReadTrackerTest.java index e2e0bf21c7..221e55c0fc 100644 --- a/test/unit/org/apache/cassandra/utils/BytesReadTrackerTest.java +++ b/test/unit/org/apache/cassandra/utils/BytesReadTrackerTest.java @@ -18,9 +18,6 @@ */ package org.apache.cassandra.utils; -import static org.junit.Assert.assertEquals; -import static org.junit.Assert.assertTrue; - import java.io.ByteArrayInputStream; import java.io.ByteArrayOutputStream; import java.io.DataInputStream; @@ -28,6 +25,9 @@ import java.io.DataOutputStream; import org.junit.Test; +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertTrue; + public class BytesReadTrackerTest { diff --git a/test/unit/org/apache/cassandra/utils/EncodedStreamsTest.java b/test/unit/org/apache/cassandra/utils/EncodedStreamsTest.java index c23ef53283..db04ddbdf7 100644 --- a/test/unit/org/apache/cassandra/utils/EncodedStreamsTest.java +++ b/test/unit/org/apache/cassandra/utils/EncodedStreamsTest.java @@ -17,31 +17,26 @@ */ package org.apache.cassandra.utils; -import static org.apache.cassandra.Util.*; +import java.io.*; -import java.io.ByteArrayInputStream; -import java.io.ByteArrayOutputStream; -import java.io.DataInputStream; -import java.io.DataOutputStream; -import java.io.IOException; +import com.google.common.collect.Iterators; +import org.junit.Assert; +import org.junit.BeforeClass; +import org.junit.Test; import org.apache.cassandra.SchemaLoader; import org.apache.cassandra.config.CFMetaData; import org.apache.cassandra.config.KSMetaData; -import org.apache.cassandra.db.ArrayBackedSortedColumns; -import org.apache.cassandra.db.ColumnFamily; +import org.apache.cassandra.db.Keyspace; +import org.apache.cassandra.db.RowUpdateBuilder; import org.apache.cassandra.db.TypeSizes; -import org.apache.cassandra.db.marshal.BytesType; -import org.apache.cassandra.db.marshal.CounterColumnType; +import org.apache.cassandra.db.rows.*; import org.apache.cassandra.exceptions.ConfigurationException; import org.apache.cassandra.locator.SimpleStrategy; import org.apache.cassandra.net.MessagingService; import org.apache.cassandra.utils.vint.EncodedDataInputStream; import org.apache.cassandra.utils.vint.EncodedDataOutputStream; -import org.junit.Assert; -import org.junit.BeforeClass; -import org.junit.Test; public class EncodedStreamsTest { @@ -55,11 +50,10 @@ public class EncodedStreamsTest { SchemaLoader.prepareServer(); SchemaLoader.createKeyspace(KEYSPACE1, - SimpleStrategy.class, - KSMetaData.optsWithRF(1), - SchemaLoader.standardCFMD(KEYSPACE1, CF_STANDARD), - SchemaLoader.standardCFMD(KEYSPACE1, CF_COUNTER) - .defaultValidator(CounterColumnType.instance)); + SimpleStrategy.class, + KSMetaData.optsWithRF(1), + SchemaLoader.standardCFMD(KEYSPACE1, CF_STANDARD), + SchemaLoader.counterCFMD(KEYSPACE1, CF_COUNTER)); } @Test @@ -112,20 +106,27 @@ public class EncodedStreamsTest Assert.assertEquals(i, idis.readLong()); } - private ColumnFamily createCF() + private UnfilteredRowIterator createTable() { - ColumnFamily cf = ArrayBackedSortedColumns.factory.create(KEYSPACE1, CF_STANDARD); - cf.addColumn(column("vijay", "try", 1)); - cf.addColumn(column("to", "be_nice", 1)); - return cf; + CFMetaData cfm = Keyspace.open(KEYSPACE1).getColumnFamilyStore(CF_STANDARD).metadata; + + RowUpdateBuilder builder = new RowUpdateBuilder(cfm, 0, "key"); + + builder.clustering("vijay").add(cfm.partitionColumns().iterator().next(), "try").build(); + builder.clustering("to").add(cfm.partitionColumns().iterator().next(), "be_nice").build(); + + return builder.unfilteredIterator(); } - private ColumnFamily createCounterCF() + private UnfilteredRowIterator createCounterTable() { - ColumnFamily cf = ArrayBackedSortedColumns.factory.create(KEYSPACE1, CF_COUNTER); - cf.addCounter(cellname("vijay"), 1); - cf.addCounter(cellname("wants"), 1000000); - return cf; + CFMetaData cfm = Keyspace.open(KEYSPACE1).getColumnFamilyStore(CF_COUNTER).metadata; + RowUpdateBuilder builder = new RowUpdateBuilder(cfm, 0, "key"); + + builder.clustering("vijay").add(cfm.partitionColumns().iterator().next(), 1L).build(); + builder.clustering("wants").add(cfm.partitionColumns().iterator().next(), 1000000L).build(); + + return builder.unfilteredIterator(); } @Test @@ -133,29 +134,27 @@ public class EncodedStreamsTest { ByteArrayOutputStream byteArrayOStream1 = new ByteArrayOutputStream(); EncodedDataOutputStream odos = new EncodedDataOutputStream(byteArrayOStream1); - ColumnFamily.serializer.serialize(createCF(), odos, version); + UnfilteredRowIteratorSerializer.serializer.serialize(createTable(), odos, version, 1); ByteArrayInputStream byteArrayIStream1 = new ByteArrayInputStream(byteArrayOStream1.toByteArray()); EncodedDataInputStream odis = new EncodedDataInputStream(new DataInputStream(byteArrayIStream1)); - ColumnFamily cf = ColumnFamily.serializer.deserialize(odis, version); - Assert.assertEquals(cf, createCF()); - Assert.assertEquals(byteArrayOStream1.size(), (int) ColumnFamily.serializer.serializedSize(cf, TypeSizes.VINT, version)); + UnfilteredRowIterator partition = UnfilteredRowIteratorSerializer.serializer.deserialize(odis, version, SerializationHelper.Flag.LOCAL); + Assert.assertTrue(Iterators.elementsEqual(partition, createTable())); + Assert.assertEquals(byteArrayOStream1.size(), (int) UnfilteredRowIteratorSerializer.serializer.serializedSize(createTable(), version, 1, TypeSizes.VINT)); } @Test public void testCounterCFSerialization() throws IOException { - ColumnFamily counterCF = createCounterCF(); - ByteArrayOutputStream byteArrayOStream1 = new ByteArrayOutputStream(); EncodedDataOutputStream odos = new EncodedDataOutputStream(byteArrayOStream1); - ColumnFamily.serializer.serialize(counterCF, odos, version); + UnfilteredRowIteratorSerializer.serializer.serialize(createCounterTable(), odos, version, 1); ByteArrayInputStream byteArrayIStream1 = new ByteArrayInputStream(byteArrayOStream1.toByteArray()); EncodedDataInputStream odis = new EncodedDataInputStream(new DataInputStream(byteArrayIStream1)); - ColumnFamily cf = ColumnFamily.serializer.deserialize(odis, version); - Assert.assertEquals(cf, counterCF); - Assert.assertEquals(byteArrayOStream1.size(), (int) ColumnFamily.serializer.serializedSize(cf, TypeSizes.VINT, version)); + UnfilteredRowIterator partition = UnfilteredRowIteratorSerializer.serializer.deserialize(odis, version, SerializationHelper.Flag.LOCAL); + Assert.assertTrue(Iterators.elementsEqual(partition, createCounterTable())); + Assert.assertEquals(byteArrayOStream1.size(), (int) UnfilteredRowIteratorSerializer.serializer.serializedSize(createCounterTable(), version, 1, TypeSizes.VINT)); } } diff --git a/test/unit/org/apache/cassandra/utils/EstimatedHistogramTest.java b/test/unit/org/apache/cassandra/utils/EstimatedHistogramTest.java index eebaa2515d..f813b9b557 100644 --- a/test/unit/org/apache/cassandra/utils/EstimatedHistogramTest.java +++ b/test/unit/org/apache/cassandra/utils/EstimatedHistogramTest.java @@ -20,7 +20,7 @@ package org.apache.cassandra.utils; import org.junit.Test; -import static org.junit.Assert.*; +import static org.junit.Assert.assertEquals; public class EstimatedHistogramTest diff --git a/test/unit/org/apache/cassandra/utils/FBUtilitiesTest.java b/test/unit/org/apache/cassandra/utils/FBUtilitiesTest.java index adf3763e9a..062387f03d 100644 --- a/test/unit/org/apache/cassandra/utils/FBUtilitiesTest.java +++ b/test/unit/org/apache/cassandra/utils/FBUtilitiesTest.java @@ -18,15 +18,16 @@ package org.apache.cassandra.utils; -import static org.junit.Assert.fail; - import java.io.IOException; import java.nio.ByteBuffer; import java.nio.charset.CharacterCodingException; import java.nio.charset.StandardCharsets; + import com.google.common.primitives.Ints; import org.junit.Test; +import static org.junit.Assert.fail; + public class FBUtilitiesTest { @Test diff --git a/test/unit/org/apache/cassandra/utils/HexTest.java b/test/unit/org/apache/cassandra/utils/HexTest.java index ad64668603..db93c089db 100644 --- a/test/unit/org/apache/cassandra/utils/HexTest.java +++ b/test/unit/org/apache/cassandra/utils/HexTest.java @@ -18,11 +18,12 @@ package org.apache.cassandra.utils; -import static org.junit.Assert.assertArrayEquals; - import java.util.Arrays; + import org.junit.Test; +import static org.junit.Assert.assertArrayEquals; + public class HexTest { @Test diff --git a/test/unit/org/apache/cassandra/utils/HistogramBuilderTest.java b/test/unit/org/apache/cassandra/utils/HistogramBuilderTest.java index dfceaf325c..4e7c439249 100644 --- a/test/unit/org/apache/cassandra/utils/HistogramBuilderTest.java +++ b/test/unit/org/apache/cassandra/utils/HistogramBuilderTest.java @@ -21,7 +21,7 @@ import java.util.concurrent.atomic.AtomicLongArray; import org.junit.Test; -import static org.junit.Assert.*; +import static org.junit.Assert.assertArrayEquals; public class HistogramBuilderTest { diff --git a/test/unit/org/apache/cassandra/utils/IntervalTreeTest.java b/test/unit/org/apache/cassandra/utils/IntervalTreeTest.java index 8409a26d72..01d7bd8b00 100644 --- a/test/unit/org/apache/cassandra/utils/IntervalTreeTest.java +++ b/test/unit/org/apache/cassandra/utils/IntervalTreeTest.java @@ -21,15 +21,16 @@ package org.apache.cassandra.utils; */ -import org.junit.Test; - +import java.io.ByteArrayInputStream; +import java.io.DataInput; +import java.io.DataInputStream; +import java.io.IOException; import java.lang.reflect.Constructor; import java.util.ArrayList; import java.util.Collections; import java.util.List; -import java.io.*; -import static org.junit.Assert.*; +import org.junit.Test; import org.apache.cassandra.db.TypeSizes; import org.apache.cassandra.io.ISerializer; @@ -37,6 +38,8 @@ import org.apache.cassandra.io.IVersionedSerializer; import org.apache.cassandra.io.util.DataOutputBuffer; import org.apache.cassandra.io.util.DataOutputPlus; +import static org.junit.Assert.assertEquals; + public class IntervalTreeTest { @Test @@ -46,27 +49,27 @@ public class IntervalTreeTest intervals.add(Interval.create(-300, -200)); intervals.add(Interval.create(-3, -2)); - intervals.add(Interval.create(1,2)); - intervals.add(Interval.create(3,6)); - intervals.add(Interval.create(2,4)); - intervals.add(Interval.create(5,7)); - intervals.add(Interval.create(1,3)); - intervals.add(Interval.create(4,6)); - intervals.add(Interval.create(8,9)); - intervals.add(Interval.create(15,20)); - intervals.add(Interval.create(40,50)); - intervals.add(Interval.create(49,60)); + intervals.add(Interval.create(1, 2)); + intervals.add(Interval.create(3, 6)); + intervals.add(Interval.create(2, 4)); + intervals.add(Interval.create(5, 7)); + intervals.add(Interval.create(1, 3)); + intervals.add(Interval.create(4, 6)); + intervals.add(Interval.create(8, 9)); + intervals.add(Interval.create(15, 20)); + intervals.add(Interval.create(40, 50)); + intervals.add(Interval.create(49, 60)); IntervalTree> it = IntervalTree.build(intervals); - assertEquals(3, it.search(Interval.create(4,4)).size()); + assertEquals(3, it.search(Interval.create(4, 4)).size()); assertEquals(4, it.search(Interval.create(4, 5)).size()); - assertEquals(7, it.search(Interval.create(-1,10)).size()); - assertEquals(0, it.search(Interval.create(-1,-1)).size()); - assertEquals(5, it.search(Interval.create(1,4)).size()); - assertEquals(2, it.search(Interval.create(0,1)).size()); - assertEquals(0, it.search(Interval.create(10,12)).size()); + assertEquals(7, it.search(Interval.create(-1, 10)).size()); + assertEquals(0, it.search(Interval.create(-1, -1)).size()); + assertEquals(5, it.search(Interval.create(1, 4)).size()); + assertEquals(2, it.search(Interval.create(0, 1)).size()); + assertEquals(0, it.search(Interval.create(10, 12)).size()); List> intervals2 = new ArrayList>(); @@ -102,16 +105,16 @@ public class IntervalTreeTest intervals.add(Interval.create(-300, -200)); intervals.add(Interval.create(-3, -2)); - intervals.add(Interval.create(1,2)); - intervals.add(Interval.create(3,6)); - intervals.add(Interval.create(2,4)); - intervals.add(Interval.create(5,7)); - intervals.add(Interval.create(1,3)); - intervals.add(Interval.create(4,6)); - intervals.add(Interval.create(8,9)); - intervals.add(Interval.create(15,20)); - intervals.add(Interval.create(40,50)); - intervals.add(Interval.create(49,60)); + intervals.add(Interval.create(1, 2)); + intervals.add(Interval.create(3, 6)); + intervals.add(Interval.create(2, 4)); + intervals.add(Interval.create(5, 7)); + intervals.add(Interval.create(1, 3)); + intervals.add(Interval.create(4, 6)); + intervals.add(Interval.create(8, 9)); + intervals.add(Interval.create(15, 20)); + intervals.add(Interval.create(40, 50)); + intervals.add(Interval.create(49, 60)); IntervalTree> it = IntervalTree.build(intervals); @@ -131,33 +134,55 @@ public class IntervalTreeTest intervals.add(Interval.create(-300, -200, "a")); intervals.add(Interval.create(-3, -2, "b")); - intervals.add(Interval.create(1,2, "c")); - intervals.add(Interval.create(1,3, "d")); - intervals.add(Interval.create(2,4, "e")); - intervals.add(Interval.create(3,6, "f")); - intervals.add(Interval.create(4,6, "g")); - intervals.add(Interval.create(5,7, "h")); - intervals.add(Interval.create(8,9, "i")); - intervals.add(Interval.create(15,20, "j")); - intervals.add(Interval.create(40,50, "k")); - intervals.add(Interval.create(49,60, "l")); + intervals.add(Interval.create(1, 2, "c")); + intervals.add(Interval.create(1, 3, "d")); + intervals.add(Interval.create(2, 4, "e")); + intervals.add(Interval.create(3, 6, "f")); + intervals.add(Interval.create(4, 6, "g")); + intervals.add(Interval.create(5, 7, "h")); + intervals.add(Interval.create(8, 9, "i")); + intervals.add(Interval.create(15, 20, "j")); + intervals.add(Interval.create(40, 50, "k")); + intervals.add(Interval.create(49, 60, "l")); IntervalTree> it = IntervalTree.build(intervals); IVersionedSerializer>> serializer = IntervalTree.serializer( - new ISerializer() - { - public void serialize(Integer i, DataOutputPlus out) throws IOException { out.writeInt(i); } - public Integer deserialize(DataInput in) throws IOException { return in.readInt(); } - public long serializedSize(Integer i, TypeSizes s) { return 4; } - }, - new ISerializer() - { - public void serialize(String v, DataOutputPlus out) throws IOException { out.writeUTF(v); } - public String deserialize(DataInput in) throws IOException { return in.readUTF(); } - public long serializedSize(String v, TypeSizes s) { return v.length(); } - }, - (Constructor>) (Object) Interval.class.getConstructor(Object.class, Object.class, Object.class) + new ISerializer() + { + public void serialize(Integer i, DataOutputPlus out) throws IOException + { + out.writeInt(i); + } + + public Integer deserialize(DataInput in) throws IOException + { + return in.readInt(); + } + + public long serializedSize(Integer i, TypeSizes s) + { + return 4; + } + }, + new ISerializer() + { + public void serialize(String v, DataOutputPlus out) throws IOException + { + out.writeUTF(v); + } + + public String deserialize(DataInput in) throws IOException + { + return in.readUTF(); + } + + public long serializedSize(String v, TypeSizes s) + { + return v.length(); + } + }, + (Constructor>) (Object) Interval.class.getConstructor(Object.class, Object.class, Object.class) ); DataOutputBuffer out = new DataOutputBuffer(); diff --git a/test/unit/org/apache/cassandra/utils/JVMStabilityInspectorTest.java b/test/unit/org/apache/cassandra/utils/JVMStabilityInspectorTest.java index f588244367..e10ffc86bd 100644 --- a/test/unit/org/apache/cassandra/utils/JVMStabilityInspectorTest.java +++ b/test/unit/org/apache/cassandra/utils/JVMStabilityInspectorTest.java @@ -17,15 +17,16 @@ */ package org.apache.cassandra.utils; -import org.apache.cassandra.config.Config; -import org.apache.cassandra.config.DatabaseDescriptor; -import org.apache.cassandra.io.FSReadError; -import org.junit.Test; - import java.io.FileNotFoundException; import java.io.IOException; import java.net.SocketException; +import org.junit.Test; + +import org.apache.cassandra.config.Config; +import org.apache.cassandra.config.DatabaseDescriptor; +import org.apache.cassandra.io.FSReadError; + import static org.junit.Assert.assertFalse; import static org.junit.Assert.assertTrue; diff --git a/test/unit/org/apache/cassandra/utils/KeyGenerator.java b/test/unit/org/apache/cassandra/utils/KeyGenerator.java index 519c5809c6..8a9d8b867b 100644 --- a/test/unit/org/apache/cassandra/utils/KeyGenerator.java +++ b/test/unit/org/apache/cassandra/utils/KeyGenerator.java @@ -18,22 +18,20 @@ */ package org.apache.cassandra.utils; -import java.io.BufferedReader; -import java.io.FileInputStream; -import java.io.FileNotFoundException; -import java.io.IOException; -import java.io.InputStreamReader; +import java.io.*; import java.nio.ByteBuffer; import java.util.Random; -public class KeyGenerator { +public class KeyGenerator +{ private static ByteBuffer randomKey(Random r) { byte[] bytes = new byte[48]; r.nextBytes(bytes); return ByteBuffer.wrap(bytes); } - static class RandomStringGenerator implements ResetableIterator { + static class RandomStringGenerator implements ResetableIterator + { int i, n, seed; Random random; @@ -66,7 +64,8 @@ public class KeyGenerator { } } - static class IntGenerator implements ResetableIterator { + static class IntGenerator implements ResetableIterator + { private int i, start, n; IntGenerator(int n) { @@ -100,7 +99,8 @@ public class KeyGenerator { } } - static class WordGenerator implements ResetableIterator { + static class WordGenerator implements ResetableIterator + { static int WORDS; static { diff --git a/test/unit/org/apache/cassandra/utils/MergeIteratorTest.java b/test/unit/org/apache/cassandra/utils/MergeIteratorTest.java index 354495592b..b6d3d58607 100644 --- a/test/unit/org/apache/cassandra/utils/MergeIteratorTest.java +++ b/test/unit/org/apache/cassandra/utils/MergeIteratorTest.java @@ -24,7 +24,6 @@ import java.util.Iterator; import com.google.common.collect.AbstractIterator; import com.google.common.collect.Iterators; import com.google.common.collect.Ordering; - import org.junit.Before; import org.junit.Test; @@ -51,9 +50,10 @@ public class MergeIteratorTest { String concatted = ""; - public void reduce(String value) + @Override + public void reduce(int idx, String current) { - concatted += value; + concatted += current; } public String getReduced() @@ -64,8 +64,8 @@ public class MergeIteratorTest } }; IMergeIterator smi = MergeIterator.get(Arrays.asList(a, b, c, d), - Ordering.natural(), - reducer); + Ordering.natural(), + reducer); assert Iterators.elementsEqual(cat, smi); smi.close(); assert a.closed && b.closed && c.closed && d.closed; diff --git a/test/unit/org/apache/cassandra/utils/MerkleTreeTest.java b/test/unit/org/apache/cassandra/utils/MerkleTreeTest.java index 8d6e272ae3..fe7f506000 100644 --- a/test/unit/org/apache/cassandra/utils/MerkleTreeTest.java +++ b/test/unit/org/apache/cassandra/utils/MerkleTreeTest.java @@ -24,12 +24,15 @@ import java.util.*; import com.google.common.collect.AbstractIterator; import com.google.common.io.ByteArrayDataInput; import com.google.common.io.ByteStreams; - import org.junit.Before; import org.junit.Test; + import org.apache.cassandra.config.DatabaseDescriptor; -import org.apache.cassandra.dht.*; +import org.apache.cassandra.dht.IPartitioner; +import org.apache.cassandra.dht.RandomPartitioner; import org.apache.cassandra.dht.RandomPartitioner.BigIntegerToken; +import org.apache.cassandra.dht.Range; +import org.apache.cassandra.dht.Token; import org.apache.cassandra.io.util.DataOutputBuffer; import org.apache.cassandra.net.MessagingService; import org.apache.cassandra.utils.MerkleTree.Hashable; diff --git a/test/unit/org/apache/cassandra/utils/SerializationsTest.java b/test/unit/org/apache/cassandra/utils/SerializationsTest.java index 497b16df1e..07752464ac 100644 --- a/test/unit/org/apache/cassandra/utils/SerializationsTest.java +++ b/test/unit/org/apache/cassandra/utils/SerializationsTest.java @@ -18,15 +18,16 @@ */ package org.apache.cassandra.utils; +import java.io.DataInputStream; +import java.io.IOException; + +import org.junit.Assert; +import org.junit.Test; + import org.apache.cassandra.AbstractSerializationsTester; import org.apache.cassandra.dht.IPartitioner; import org.apache.cassandra.io.util.DataOutputStreamPlus; import org.apache.cassandra.service.StorageService; -import org.junit.Assert; -import org.junit.Test; - -import java.io.DataInputStream; -import java.io.IOException; public class SerializationsTest extends AbstractSerializationsTester { diff --git a/test/unit/org/apache/cassandra/utils/StreamingHistogramTest.java b/test/unit/org/apache/cassandra/utils/StreamingHistogramTest.java index 0e9b90b0f8..38b2f04929 100644 --- a/test/unit/org/apache/cassandra/utils/StreamingHistogramTest.java +++ b/test/unit/org/apache/cassandra/utils/StreamingHistogramTest.java @@ -17,11 +17,13 @@ */ package org.apache.cassandra.utils; -import org.junit.Test; - import java.io.ByteArrayInputStream; import java.io.DataInputStream; -import java.util.*; +import java.util.Iterator; +import java.util.LinkedHashMap; +import java.util.Map; + +import org.junit.Test; import org.apache.cassandra.io.util.DataOutputBuffer; diff --git a/test/unit/org/apache/cassandra/utils/TopKSamplerTest.java b/test/unit/org/apache/cassandra/utils/TopKSamplerTest.java index cad48f48de..db7ae39693 100644 --- a/test/unit/org/apache/cassandra/utils/TopKSamplerTest.java +++ b/test/unit/org/apache/cassandra/utils/TopKSamplerTest.java @@ -2,18 +2,19 @@ package org.apache.cassandra.utils; import java.util.List; import java.util.Map; -import java.util.concurrent.*; +import java.util.concurrent.CountDownLatch; +import java.util.concurrent.TimeUnit; +import java.util.concurrent.TimeoutException; import java.util.concurrent.atomic.AtomicBoolean; -import junit.framework.Assert; - -import org.apache.cassandra.utils.TopKSampler.SamplerResult; +import com.google.common.collect.Maps; +import com.google.common.util.concurrent.Uninterruptibles; import org.junit.Test; import com.clearspring.analytics.hash.MurmurHash; import com.clearspring.analytics.stream.Counter; -import com.google.common.collect.Maps; -import com.google.common.util.concurrent.Uninterruptibles; +import junit.framework.Assert; +import org.apache.cassandra.utils.TopKSampler.SamplerResult; public class TopKSamplerTest { diff --git a/test/unit/org/apache/cassandra/utils/UUIDTests.java b/test/unit/org/apache/cassandra/utils/UUIDTests.java index 99cd5ae9ce..83e421ad77 100644 --- a/test/unit/org/apache/cassandra/utils/UUIDTests.java +++ b/test/unit/org/apache/cassandra/utils/UUIDTests.java @@ -21,12 +21,14 @@ package org.apache.cassandra.utils; */ -import org.apache.cassandra.db.marshal.TimeUUIDType; -import org.junit.Test; - import java.nio.ByteBuffer; import java.util.UUID; +import org.junit.Test; + +import org.apache.cassandra.db.marshal.TimeUUIDType; +import org.apache.cassandra.utils.UUIDGen; + public class UUIDTests { diff --git a/test/unit/org/apache/cassandra/utils/memory/NativeAllocatorTest.java b/test/unit/org/apache/cassandra/utils/memory/NativeAllocatorTest.java index 83d6c0c844..354c9b8384 100644 --- a/test/unit/org/apache/cassandra/utils/memory/NativeAllocatorTest.java +++ b/test/unit/org/apache/cassandra/utils/memory/NativeAllocatorTest.java @@ -22,7 +22,6 @@ import java.util.concurrent.*; import java.util.concurrent.atomic.AtomicReference; import com.google.common.util.concurrent.Uninterruptibles; - import org.junit.Test; import junit.framework.Assert; diff --git a/tools/bin/json2sstable b/tools/bin/json2sstable deleted file mode 100755 index a0278d43c0..0000000000 --- a/tools/bin/json2sstable +++ /dev/null @@ -1,51 +0,0 @@ -#!/bin/sh - -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -if [ "x$CASSANDRA_INCLUDE" = "x" ]; then - for include in "`dirname "$0"`/cassandra.in.sh" \ - "$HOME/.cassandra.in.sh" \ - /usr/share/cassandra/cassandra.in.sh \ - /usr/local/share/cassandra/cassandra.in.sh \ - /opt/cassandra/cassandra.in.sh; do - if [ -r "$include" ]; then - . "$include" - break - fi - done -elif [ -r "$CASSANDRA_INCLUDE" ]; then - . "$CASSANDRA_INCLUDE" -fi - -# Use JAVA_HOME if set, otherwise look for java in PATH -if [ -x "$JAVA_HOME/bin/java" ]; then - JAVA="$JAVA_HOME/bin/java" -else - JAVA="`which java`" -fi - -if [ -z "$CLASSPATH" ]; then - echo "You must set the CLASSPATH var" >&2 - exit 1 -fi - -"$JAVA" $JAVA_AGENT -cp "$CLASSPATH" $JVM_OPTS -Dstorage-config="$CASSANDRA_CONF" \ - -Dcassandra.storagedir="$cassandra_storagedir" \ - -Dlogback.configurationFile=logback-tools.xml \ - org.apache.cassandra.tools.SSTableImport "$@" - -# vi:ai sw=4 ts=4 tw=0 et diff --git a/tools/bin/json2sstable.bat b/tools/bin/json2sstable.bat deleted file mode 100644 index db0fa91611..0000000000 --- a/tools/bin/json2sstable.bat +++ /dev/null @@ -1,48 +0,0 @@ -@REM -@REM Licensed to the Apache Software Foundation (ASF) under one or more -@REM contributor license agreements. See the NOTICE file distributed with -@REM this work for additional information regarding copyright ownership. -@REM The ASF licenses this file to You under the Apache License, Version 2.0 -@REM (the "License"); you may not use this file except in compliance with -@REM the License. You may obtain a copy of the License at -@REM -@REM http://www.apache.org/licenses/LICENSE-2.0 -@REM -@REM Unless required by applicable law or agreed to in writing, software -@REM distributed under the License is distributed on an "AS IS" BASIS, -@REM WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -@REM See the License for the specific language governing permissions and -@REM limitations under the License. - -@echo off -if "%OS%" == "Windows_NT" setlocal - -pushd "%~dp0" -call cassandra.in.bat - -if NOT DEFINED CASSANDRA_MAIN set CASSANDRA_MAIN=org.apache.cassandra.tools.SSTableImport -if NOT DEFINED JAVA_HOME goto :err - -REM ***** JAVA options ***** -set JAVA_OPTS=^ - -Dlogback.configurationFile=logback-tools.xml - -set TOOLS_PARAMS= -FOR %%A IN (%*) DO call :appendToolsParams %%A -goto runTool - -:appendToolsParams -set TOOLS_PARAMS=%TOOLS_PARAMS% %1 -goto :eof - -:runTool -"%JAVA_HOME%\bin\java" %JAVA_OPTS% %CASSANDRA_PARAMS% -cp %CASSANDRA_CLASSPATH% "%CASSANDRA_MAIN%" %TOOLS_PARAMS% -goto finally - -:err -echo JAVA_HOME environment variable must be set! -pause - -:finally - -ENDLOCAL diff --git a/tools/bin/sstable2json b/tools/bin/sstable2json deleted file mode 100755 index 7eeb708824..0000000000 --- a/tools/bin/sstable2json +++ /dev/null @@ -1,52 +0,0 @@ -#!/bin/sh - -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -if [ "x$CASSANDRA_INCLUDE" = "x" ]; then - for include in "`dirname "$0"`/cassandra.in.sh" \ - "$HOME/.cassandra.in.sh" \ - /usr/share/cassandra/cassandra.in.sh \ - /usr/local/share/cassandra/cassandra.in.sh \ - /opt/cassandra/cassandra.in.sh; do - if [ -r "$include" ]; then - . "$include" - break - fi - done -elif [ -r "$CASSANDRA_INCLUDE" ]; then - . "$CASSANDRA_INCLUDE" -fi - - -# Use JAVA_HOME if set, otherwise look for java in PATH -if [ -x "$JAVA_HOME/bin/java" ]; then - JAVA="$JAVA_HOME/bin/java" -else - JAVA="`which java`" -fi - -if [ -z "$CLASSPATH" ]; then - echo "You must set the CLASSPATH var" >&2 - exit 1 -fi - -"$JAVA" $JAVA_AGENT -cp "$CLASSPATH" $JVM_OPTS -Dstorage-config="$CASSANDRA_CONF" \ - -Dcassandra.storagedir="$cassandra_storagedir" \ - -Dlogback.configurationFile=logback-tools.xml \ - org.apache.cassandra.tools.SSTableExport "$@" - -# vi:ai sw=4 ts=4 tw=0 et diff --git a/tools/bin/sstable2json.bat b/tools/bin/sstable2json.bat deleted file mode 100644 index 17669c0313..0000000000 --- a/tools/bin/sstable2json.bat +++ /dev/null @@ -1,48 +0,0 @@ -@REM -@REM Licensed to the Apache Software Foundation (ASF) under one or more -@REM contributor license agreements. See the NOTICE file distributed with -@REM this work for additional information regarding copyright ownership. -@REM The ASF licenses this file to You under the Apache License, Version 2.0 -@REM (the "License"); you may not use this file except in compliance with -@REM the License. You may obtain a copy of the License at -@REM -@REM http://www.apache.org/licenses/LICENSE-2.0 -@REM -@REM Unless required by applicable law or agreed to in writing, software -@REM distributed under the License is distributed on an "AS IS" BASIS, -@REM WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -@REM See the License for the specific language governing permissions and -@REM limitations under the License. - -@echo off -if "%OS%" == "Windows_NT" setlocal - -pushd "%~dp0" -call cassandra.in.bat - -if NOT DEFINED CASSANDRA_MAIN set CASSANDRA_MAIN=org.apache.cassandra.tools.SSTableExport -if NOT DEFINED JAVA_HOME goto :err - -REM ***** JAVA options ***** -set JAVA_OPTS=^ - -Dlogback.configurationFile=logback-tools.xml - -set TOOLS_PARAMS= -FOR %%A IN (%*) DO call :appendToolsParams %%A -goto runTool - -:appendToolsParams -set TOOLS_PARAMS=%TOOLS_PARAMS% %1 -goto :eof - -:runTool -"%JAVA_HOME%\bin\java" %JAVA_OPTS% %CASSANDRA_PARAMS% -cp %CASSANDRA_CLASSPATH% "%CASSANDRA_MAIN%" %TOOLS_PARAMS% -goto finally - -:err -echo JAVA_HOME environment variable must be set! -pause - -:finally - -ENDLOCAL