natas_hw
/
cassandra
mirror of https://github.com/apache/cassandra
0
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

Add sstabledump tool 

patch by Chris Lohfink; reviewed by yukim for CASSANDRA-7464
This commit is contained in:
Chris Lohfink 2016-02-23 11:22:26 -06:00 committed by Yuki Morishita
parent a62397782c
commit 71b1c4a631
15 changed files with 933 additions and 9 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

1
CHANGES.txt
View File

@ -1,4 +1,5 @@
3.0.4
* Add sstabledump tool (CASSANDRA-7464)
* Introduce backpressure for hints (CASSANDRA-10972)
* Fix ClusteringPrefix not being able to read tombstone range boundaries (CASSANDRA-11158)
* Prevent logging in sandboxed state (CASSANDRA-11033)

14
NEWS.txt
View File

@ -13,6 +13,20 @@ restore snapshots created with the previous major version using the
'sstableloader' tool. You can upgrade the file format of your snapshots
using the provided 'sstableupgrade' tool.
3.0.4
=====
New features
------------
- sstabledump tool is added to be 3.0 version of former sstable2json. The tool only
supports v3.0+ SSTables. See tool's help for more detail.
Upgrading
---------
- Nothing specific to this release, but please see previous versions upgrading section,
especially if you are upgrading from 2.2.
3.0.3
=====

8
src/java/org/apache/cassandra/config/CFMetaData.java
View File

@ -1132,7 +1132,7 @@ public final class CFMetaData
private final boolean isSuper;
private final boolean isCounter;
private final boolean isView;
private IPartitioner partitioner;
private Optional<IPartitioner> partitioner;
private UUID tableId;
@ -1150,7 +1150,7 @@ public final class CFMetaData
this.isSuper = isSuper;
this.isCounter = isCounter;
this.isView = isView;
this.partitioner = DatabaseDescriptor.getPartitioner();
this.partitioner = Optional.empty();
}
public static Builder create(String keyspace, String table)
@ -1185,7 +1185,7 @@ public final class CFMetaData
public Builder withPartitioner(IPartitioner partitioner)
{
this.partitioner = partitioner;
this.partitioner = Optional.ofNullable(partitioner);
return this;
}
@ -1296,7 +1296,7 @@ public final class CFMetaData
partitions,
clusterings,
builder.build(),
partitioner);
partitioner.orElseGet(DatabaseDescriptor::getPartitioner));
}
}

25
src/java/org/apache/cassandra/db/SerializationHeader.java
View File

@ -361,6 +361,31 @@ public class SerializationHeader
return String.format("SerializationHeader.Component[key=%s, cks=%s, statics=%s, regulars=%s, stats=%s]",
keyType, clusteringTypes, staticColumns, regularColumns, stats);
}
public AbstractType<?> getKetType()
{
return keyType;
}
public List<AbstractType<?>> getClusteringTypes()
{
return clusteringTypes;
}
public Map<ByteBuffer, AbstractType<?>> getStaticColumns()
{
return staticColumns;
}
public Map<ByteBuffer, AbstractType<?>> getRegularColumns()
{
return regularColumns;
}
public EncodingStats getEncodingStats()
{
return stats;
}
}
public static class Serializer implements IMetadataComponentSerializer<Component>

4
src/java/org/apache/cassandra/db/rows/AbstractRangeTombstoneMarker.java
View File

@ -71,4 +71,8 @@ public abstract class AbstractRangeTombstoneMarker implements RangeTombstoneMark
{
return toString(metadata);
}
public String toString(CFMetaData metadata, boolean includeClusteringKeys, boolean fullDetails)
{
return toString(metadata);
}
}

12
src/java/org/apache/cassandra/db/rows/AbstractRow.java
View File

@ -91,6 +91,11 @@ public abstract class AbstractRow extends AbstractCollection<ColumnData> impleme
}
public String toString(CFMetaData metadata, boolean fullDetails)
{
return toString(metadata, true, fullDetails);
}
public String toString(CFMetaData metadata, boolean includeClusterKeys, boolean fullDetails)
{
StringBuilder sb = new StringBuilder();
sb.append("Row");
@ -101,7 +106,12 @@ public abstract class AbstractRow extends AbstractCollection<ColumnData> impleme
sb.append(" del=").append(deletion());
sb.append(" ]");
}
sb.append(": ").append(clustering().toString(metadata)).append(" | ");
sb.append(": ");
if(includeClusterKeys)
sb.append(clustering().toString(metadata));
else
sb.append(clustering().toCQLString(metadata));
sb.append(" | ");
boolean isFirst = true;
for (ColumnData cd : this)
{

1
src/java/org/apache/cassandra/db/rows/Unfiltered.java
View File

@ -57,6 +57,7 @@ public interface Unfiltered extends Clusterable
public String toString(CFMetaData metadata);
public String toString(CFMetaData metadata, boolean fullDetails);
public String toString(CFMetaData metadata, boolean includeClusterKeys, boolean fullDetails);
default boolean isRow()
{

8
src/java/org/apache/cassandra/io/sstable/format/SSTableReader.java
View File

@ -1760,6 +1760,14 @@ public abstract class SSTableReader extends SSTable implements SelfRefCounted<SS
*/
public abstract ISSTableScanner getScanner(Collection<Range<Token>> ranges, RateLimiter limiter);
/**
* Direct I/O SSTableScanner over an iterator of bounds.
*
* @param bounds the keys to cover
* @return A Scanner for seeking over the rows of the SSTable.
*/
public abstract ISSTableScanner getScanner(Iterator<AbstractBounds<PartitionPosition>> rangeIterator);
/**
* @param columns the columns to return.
* @param dataRange filter to use when reading the columns

13
src/java/org/apache/cassandra/io/sstable/format/big/BigTableReader.java
View File

@ -25,7 +25,7 @@ import org.apache.cassandra.db.rows.SliceableUnfilteredRowIterator;
import org.apache.cassandra.db.filter.ColumnFilter;
import org.apache.cassandra.db.columniterator.SSTableIterator;
import org.apache.cassandra.db.columniterator.SSTableReversedIterator;
import org.apache.cassandra.dht.IPartitioner;
import org.apache.cassandra.dht.AbstractBounds;
import org.apache.cassandra.dht.Range;
import org.apache.cassandra.dht.Token;
import org.apache.cassandra.io.sstable.Component;
@ -81,6 +81,17 @@ public class BigTableReader extends SSTableReader
return BigTableScanner.getScanner(this, columns, dataRange, limiter, isForThrift);
}
/**
* Direct I/O SSTableScanner over an iterator of bounds.
*
* @param boundsIterator the keys to cover
* @return A Scanner for seeking over the rows of the SSTable.
*/
public ISSTableScanner getScanner(Iterator<AbstractBounds<PartitionPosition>> boundsIterator)
{
return BigTableScanner.getScanner(this, boundsIterator);
}
/**
* Direct I/O SSTableScanner over the full sstable.
*

5
src/java/org/apache/cassandra/io/sstable/format/big/BigTableScanner.java
View File

@ -86,6 +86,11 @@ public class BigTableScanner implements ISSTableScanner
return new BigTableScanner(sstable, ColumnFilter.all(sstable.metadata), null, limiter, false, makeBounds(sstable, tokenRanges).iterator());
}
public static ISSTableScanner getScanner(SSTableReader sstable, Iterator<AbstractBounds<PartitionPosition>> rangeIterator)
{
return new BigTableScanner(sstable, ColumnFilter.all(sstable.metadata), null, null, false, rangeIterator);
}
private BigTableScanner(SSTableReader sstable, ColumnFilter columns, DataRange dataRange, RateLimiter limiter, boolean isForThrift, Iterator<AbstractBounds<PartitionPosition>> rangeIterator)
{
assert sstable != null;

501
src/java/org/apache/cassandra/tools/JsonTransformer.java Normal file
View File

@ -0,0 +1,501 @@
package org.apache.cassandra.tools;
import java.io.IOException;
import java.io.OutputStream;
import java.io.OutputStreamWriter;
import java.nio.ByteBuffer;
import java.util.List;
import java.util.stream.Stream;
import org.apache.cassandra.config.CFMetaData;
import org.apache.cassandra.config.ColumnDefinition;
import org.apache.cassandra.db.ClusteringPrefix;
import org.apache.cassandra.db.DecoratedKey;
import org.apache.cassandra.db.DeletionTime;
import org.apache.cassandra.db.LivenessInfo;
import org.apache.cassandra.db.RangeTombstone;
import org.apache.cassandra.db.marshal.AbstractType;
import org.apache.cassandra.db.marshal.CollectionType;
import org.apache.cassandra.db.marshal.CompositeType;
import org.apache.cassandra.db.rows.Cell;
import org.apache.cassandra.db.rows.RangeTombstoneBoundMarker;
import org.apache.cassandra.db.rows.RangeTombstoneBoundaryMarker;
import org.apache.cassandra.db.rows.RangeTombstoneMarker;
import org.apache.cassandra.db.rows.Row;
import org.apache.cassandra.db.rows.Unfiltered;
import org.apache.cassandra.db.rows.UnfilteredRowIterator;
import org.apache.cassandra.io.sstable.ISSTableScanner;
import org.apache.cassandra.utils.ByteBufferUtil;
import org.codehaus.jackson.JsonFactory;
import org.codehaus.jackson.JsonGenerator;
import org.codehaus.jackson.impl.Indenter;
import org.codehaus.jackson.util.DefaultPrettyPrinter;
import org.codehaus.jackson.util.DefaultPrettyPrinter.NopIndenter;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
public final class JsonTransformer
{
private static final Logger logger = LoggerFactory.getLogger(JsonTransformer.class);
private static final JsonFactory jsonFactory = new JsonFactory();
private final JsonGenerator json;
private final CompactIndenter objectIndenter = new CompactIndenter();
private final CompactIndenter arrayIndenter = new CompactIndenter();
private final CFMetaData metadata;
private final ISSTableScanner currentScanner;
private long currentPosition = 0;
private JsonTransformer(JsonGenerator json, ISSTableScanner currentScanner, CFMetaData metadata)
{
this.json = json;
this.metadata = metadata;
this.currentScanner = currentScanner;
DefaultPrettyPrinter prettyPrinter = new DefaultPrettyPrinter();
prettyPrinter.indentObjectsWith(objectIndenter);
prettyPrinter.indentArraysWith(arrayIndenter);
json.setPrettyPrinter(prettyPrinter);
}
public static void toJson(ISSTableScanner currentScanner, Stream<UnfilteredRowIterator> partitions, CFMetaData metadata, OutputStream out)
throws IOException
{
try (JsonGenerator json = jsonFactory.createJsonGenerator(new OutputStreamWriter(out, "UTF-8")))
{
JsonTransformer transformer = new JsonTransformer(json, currentScanner, metadata);
json.writeStartArray();
partitions.forEach(transformer::serializePartition);
json.writeEndArray();
}
}
public static void keysToJson(ISSTableScanner currentScanner, Stream<DecoratedKey> keys, CFMetaData metadata, OutputStream out) throws IOException
{
try (JsonGenerator json = jsonFactory.createJsonGenerator(new OutputStreamWriter(out, "UTF-8")))
{
JsonTransformer transformer = new JsonTransformer(json, currentScanner, metadata);
json.writeStartArray();
keys.forEach(transformer::serializePartitionKey);
json.writeEndArray();
}
}
private void updatePosition()
{
this.currentPosition = currentScanner.getCurrentPosition();
}
private void serializePartitionKey(DecoratedKey key)
{
AbstractType<?> keyValidator = metadata.getKeyValidator();
objectIndenter.setCompact(true);
try
{
arrayIndenter.setCompact(true);
json.writeStartArray();
if (keyValidator instanceof CompositeType)
{
// if a composite type, the partition has multiple keys.
CompositeType compositeType = (CompositeType) keyValidator;
ByteBuffer keyBytes = key.getKey().duplicate();
// Skip static data if it exists.
if (keyBytes.remaining() >= 2)
{
int header = ByteBufferUtil.getShortLength(keyBytes, keyBytes.position());
if ((header & 0xFFFF) == 0xFFFF)
{
ByteBufferUtil.readShortLength(keyBytes);
}
}
int i = 0;
while (keyBytes.remaining() > 0 && i < compositeType.getComponents().size())
{
AbstractType<?> colType = compositeType.getComponents().get(i);
ByteBuffer value = ByteBufferUtil.readBytesWithShortLength(keyBytes);
String colValue = colType.getString(value);
json.writeString(colValue);
byte b = keyBytes.get();
if (b != 0)
{
break;
}
++i;
}
}
else
{
// if not a composite type, assume a single column partition key.
assert metadata.partitionKeyColumns().size() == 1;
json.writeString(keyValidator.getString(key.getKey()));
}
json.writeEndArray();
objectIndenter.setCompact(false);
arrayIndenter.setCompact(false);
}
catch (IOException e)
{
logger.error("Failure serializing partition key.", e);
}
}
private void serializePartition(UnfilteredRowIterator partition)
{
String key = metadata.getKeyValidator().getString(partition.partitionKey().getKey());
try
{
json.writeStartObject();
json.writeFieldName("partition");
json.writeStartObject();
json.writeFieldName("key");
serializePartitionKey(partition.partitionKey());
json.writeNumberField("position", this.currentScanner.getCurrentPosition());
if (!partition.partitionLevelDeletion().isLive())
{
json.writeFieldName("deletion_info");
objectIndenter.setCompact(true);
json.writeStartObject();
json.writeFieldName("deletion_time");
json.writeNumber(partition.partitionLevelDeletion().markedForDeleteAt());
json.writeFieldName("tstamp");
json.writeNumber(partition.partitionLevelDeletion().localDeletionTime());
json.writeEndObject();
objectIndenter.setCompact(false);
json.writeEndObject();
}
else
{
json.writeEndObject();
json.writeFieldName("rows");
json.writeStartArray();
updatePosition();
if (!partition.staticRow().isEmpty())
{
serializeRow(partition.staticRow());
}
Unfiltered unfiltered;
updatePosition();
while (partition.hasNext())
{
unfiltered = partition.next();
if (unfiltered instanceof Row)
{
serializeRow((Row) unfiltered);
}
else if (unfiltered instanceof RangeTombstoneMarker)
{
serializeTombstone((RangeTombstoneMarker) unfiltered);
}
updatePosition();
}
json.writeEndArray();
}
json.writeEndObject();
}
catch (IOException e)
{
logger.error("Fatal error parsing partition: {}", key, e);
}
}
private void serializeRow(Row row)
{
try
{
json.writeStartObject();
String rowType = row.isStatic() ? "static_block" : "row";
json.writeFieldName("type");
json.writeString(rowType);
json.writeNumberField("position", this.currentPosition);
// Only print clustering information for non-static rows.
if (!row.isStatic())
{
serializeClustering(row.clustering());
}
LivenessInfo liveInfo = row.primaryKeyLivenessInfo();
if (!liveInfo.isEmpty())
{
objectIndenter.setCompact(false);
json.writeFieldName("liveness_info");
objectIndenter.setCompact(true);
json.writeStartObject();
json.writeFieldName("tstamp");
json.writeNumber(liveInfo.timestamp());
if (liveInfo.isExpiring())
{
json.writeFieldName("ttl");
json.writeNumber(liveInfo.ttl());
json.writeFieldName("expires_at");
json.writeNumber(liveInfo.localExpirationTime());
json.writeFieldName("expired");
json.writeBoolean(liveInfo.localExpirationTime() < (System.currentTimeMillis() / 1000));
}
json.writeEndObject();
objectIndenter.setCompact(false);
}
// If this is a deletion, indicate that, otherwise write cells.
if (!row.deletion().isLive())
{
json.writeFieldName("deletion_info");
objectIndenter.setCompact(true);
json.writeStartObject();
json.writeFieldName("deletion_time");
json.writeNumber(row.deletion().time().markedForDeleteAt());
json.writeFieldName("tstamp");
json.writeNumber(row.deletion().time().localDeletionTime());
json.writeEndObject();
objectIndenter.setCompact(false);
}
else
{
json.writeFieldName("cells");
json.writeStartArray();
row.cells().forEach(c -> serializeCell(c, liveInfo));
json.writeEndArray();
}
json.writeEndObject();
}
catch (IOException e)
{
logger.error("Fatal error parsing row.", e);
}
}
private void serializeTombstone(RangeTombstoneMarker tombstone)
{
try
{
json.writeStartObject();
json.writeFieldName("type");
if (tombstone instanceof RangeTombstoneBoundMarker)
{
json.writeString("range_tombstone_bound");
RangeTombstoneBoundMarker bm = (RangeTombstoneBoundMarker) tombstone;
serializeBound(bm.clustering(), bm.deletionTime());
}
else
{
assert tombstone instanceof RangeTombstoneBoundaryMarker;
json.writeString("range_tombstone_boundary");
RangeTombstoneBoundaryMarker bm = (RangeTombstoneBoundaryMarker) tombstone;
serializeBound(bm.openBound(false), bm.openDeletionTime(false));
serializeBound(bm.closeBound(false), bm.closeDeletionTime(false));
}
json.writeEndObject();
objectIndenter.setCompact(false);
}
catch (IOException e)
{
logger.error("Failure parsing tombstone.", e);
}
}
private void serializeBound(RangeTombstone.Bound bound, DeletionTime deletionTime) throws IOException
{
json.writeFieldName(bound.isStart() ? "start" : "end");
json.writeStartObject();
json.writeFieldName("type");
json.writeString(bound.isInclusive() ? "inclusive" : "exclusive");
serializeClustering(bound.clustering());
serializeDeletion(deletionTime);
json.writeEndObject();
}
private void serializeClustering(ClusteringPrefix clustering) throws IOException
{
if (clustering.size() > 0)
{
json.writeFieldName("clustering");
objectIndenter.setCompact(true);
json.writeStartArray();
arrayIndenter.setCompact(true);
List<ColumnDefinition> clusteringColumns = metadata.clusteringColumns();
for (int i = 0; i < clusteringColumns.size(); i++)
{
ColumnDefinition column = clusteringColumns.get(i);
if (i >= clustering.size())
{
json.writeString("*");
}
else
{
json.writeString(column.cellValueType().getString(clustering.get(i)));
}
}
json.writeEndArray();
objectIndenter.setCompact(false);
arrayIndenter.setCompact(false);
}
}
private void serializeDeletion(DeletionTime deletion) throws IOException
{
json.writeFieldName("deletion_info");
objectIndenter.setCompact(true);
json.writeStartObject();
json.writeFieldName("deletion_time");
json.writeNumber(deletion.markedForDeleteAt());
json.writeFieldName("tstamp");
json.writeNumber(deletion.localDeletionTime());
json.writeEndObject();
objectIndenter.setCompact(false);
}
private void serializeCell(Cell cell, LivenessInfo liveInfo)
{
try
{
json.writeStartObject();
objectIndenter.setCompact(true);
json.writeFieldName("name");
AbstractType<?> type = cell.column().type;
json.writeString(cell.column().name.toCQLString());
if (cell.path() != null && cell.path().size() > 0)
{
CollectionType ct = (CollectionType) type;
json.writeFieldName("path");
arrayIndenter.setCompact(true);
json.writeStartArray();
for (int i = 0; i < cell.path().size(); i++)
{
json.writeString(ct.nameComparator().getString(cell.path().get(i)));
}
json.writeEndArray();
arrayIndenter.setCompact(false);
}
if (cell.isTombstone())
{
json.writeFieldName("deletion_time");
json.writeNumber(cell.localDeletionTime());
}
else
{
json.writeFieldName("value");
json.writeString(cell.column().cellValueType().getString(cell.value()));
}
if (liveInfo.isEmpty() || cell.timestamp() != liveInfo.timestamp())
{
json.writeFieldName("tstamp");
json.writeNumber(cell.timestamp());
}
if (cell.isExpiring() && (liveInfo.isEmpty() || cell.ttl() != liveInfo.ttl()))
{
json.writeFieldName("ttl");
json.writeNumber(cell.ttl());
json.writeFieldName("expires_at");
json.writeNumber(cell.localDeletionTime());
json.writeFieldName("expired");
json.writeBoolean(!cell.isLive((int) (System.currentTimeMillis() / 1000)));
}
json.writeEndObject();
objectIndenter.setCompact(false);
}
catch (IOException e)
{
logger.error("Failure parsing cell.", e);
}
}
/**
* A specialized {@link Indenter} that enables a 'compact' mode which puts all subsequent json values on the same
* line. This is manipulated via {@link CompactIndenter#setCompact(boolean)}
*/
private static final class CompactIndenter extends NopIndenter
{
private static final int INDENT_LEVELS = 16;
private final char[] indents;
private final int charsPerLevel;
private final String eol;
private static final String space = " ";
private boolean compact = false;
CompactIndenter()
{
this(" ", System.lineSeparator());
}
CompactIndenter(String indent, String eol)
{
this.eol = eol;
charsPerLevel = indent.length();
indents = new char[indent.length() * INDENT_LEVELS];
int offset = 0;
for (int i = 0; i < INDENT_LEVELS; i++)
{
indent.getChars(0, indent.length(), indents, offset);
offset += indent.length();
}
}
@Override
public boolean isInline()
{
return false;
}
/**
* Configures whether or not subsequent json values should be on the same line delimited by string or not.
*
* @param compact
* Whether or not to compact.
*/
public void setCompact(boolean compact)
{
this.compact = compact;
}
@Override
public void writeIndentation(JsonGenerator jg, int level)
{
try
{
if (!compact)
{
jg.writeRaw(eol);
if (level > 0)
{ // should we err on negative values (as there's some flaw?)
level *= charsPerLevel;
while (level > indents.length)
{ // unlike to happen but just in case
jg.writeRaw(indents, 0, indents.length);
level -= indents.length;
}
jg.writeRaw(indents, 0, level);
}
}
else
{
jg.writeRaw(space);
}
}
catch (IOException e)
{
e.printStackTrace();
System.exit(1);
}
}
}
}

242
src/java/org/apache/cassandra/tools/SSTableExport.java Normal file
View File

@ -0,0 +1,242 @@
/*
* Licensed to the Apache Software Foundation (ASF) under one
* or more contributor license agreements. See the NOTICE file
* distributed with this work for additional information
* regarding copyright ownership. The ASF licenses this file
* to you under the Apache License, Version 2.0 (the
* "License"); you may not use this file except in compliance
* with the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.cassandra.tools;
import java.io.File;
import java.io.IOException;
import java.util.*;
import java.util.concurrent.atomic.AtomicLong;
import java.util.stream.Collectors;
import java.util.stream.Stream;
import java.util.stream.StreamSupport;
import org.apache.commons.cli.*;
import org.apache.cassandra.config.CFMetaData;
import org.apache.cassandra.config.Config;
import org.apache.cassandra.cql3.ColumnIdentifier;
import org.apache.cassandra.db.DecoratedKey;
import org.apache.cassandra.db.PartitionPosition;
import org.apache.cassandra.db.SerializationHeader;
import org.apache.cassandra.db.marshal.UTF8Type;
import org.apache.cassandra.db.rows.UnfilteredRowIterator;
import org.apache.cassandra.dht.AbstractBounds;
import org.apache.cassandra.dht.Bounds;
import org.apache.cassandra.dht.IPartitioner;
import org.apache.cassandra.exceptions.ConfigurationException;
import org.apache.cassandra.io.sstable.Descriptor;
import org.apache.cassandra.io.sstable.ISSTableScanner;
import org.apache.cassandra.io.sstable.KeyIterator;
import org.apache.cassandra.io.sstable.format.SSTableReader;
import org.apache.cassandra.io.sstable.metadata.MetadataComponent;
import org.apache.cassandra.io.sstable.metadata.MetadataType;
import org.apache.cassandra.io.sstable.metadata.ValidationMetadata;
import org.apache.cassandra.utils.FBUtilities;
/**
* Export SSTables to JSON format.
*/
public class SSTableExport
{
private static final String KEY_OPTION = "k";
private static final String DEBUG_OUTPUT_OPTION = "d";
private static final String EXCLUDE_KEY_OPTION = "x";
private static final String ENUMERATE_KEYS_OPTION = "e";
private static final Options options = new Options();
private static CommandLine cmd;
static
{
Config.setClientMode(true);
Option optKey = new Option(KEY_OPTION, true, "Row key");
// Number of times -k <key> can be passed on the command line.
optKey.setArgs(500);
options.addOption(optKey);
Option excludeKey = new Option(EXCLUDE_KEY_OPTION, true, "Excluded row key");
// Number of times -x <key> can be passed on the command line.
excludeKey.setArgs(500);
options.addOption(excludeKey);
Option optEnumerate = new Option(ENUMERATE_KEYS_OPTION, false, "enumerate keys only");
options.addOption(optEnumerate);
Option debugOutput = new Option(DEBUG_OUTPUT_OPTION, false, "CQL row per line internal representation");
options.addOption(debugOutput);
}
/**
* Construct table schema from info stored in SSTable's Stats.db
*
* @param desc SSTable's descriptor
* @return Restored CFMetaData
* @throws IOException when Stats.db cannot be read
*/
public static CFMetaData metadataFromSSTable(Descriptor desc) throws IOException
{
if (!desc.version.storeRows())
throw new IOException("pre-3.0 SSTable is not supported.");
EnumSet<MetadataType> types = EnumSet.of(MetadataType.VALIDATION, MetadataType.STATS, MetadataType.HEADER);
Map<MetadataType, MetadataComponent> sstableMetadata = desc.getMetadataSerializer().deserialize(desc, types);
ValidationMetadata validationMetadata = (ValidationMetadata) sstableMetadata.get(MetadataType.VALIDATION);
SerializationHeader.Component header = (SerializationHeader.Component) sstableMetadata.get(MetadataType.HEADER);
IPartitioner partitioner = FBUtilities.newPartitioner(validationMetadata.partitioner);
CFMetaData.Builder builder = CFMetaData.Builder.create("keyspace", "table").withPartitioner(partitioner);
header.getStaticColumns().entrySet().stream()
.forEach(entry -> {
ColumnIdentifier ident = ColumnIdentifier.getInterned(UTF8Type.instance.getString(entry.getKey()), true);
builder.addStaticColumn(ident, entry.getValue());
});
header.getRegularColumns().entrySet().stream()
.forEach(entry -> {
ColumnIdentifier ident = ColumnIdentifier.getInterned(UTF8Type.instance.getString(entry.getKey()), true);
builder.addRegularColumn(ident, entry.getValue());
});
builder.addPartitionKey("PartitionKey", header.getKetType());
for (int i = 0; i < header.getClusteringTypes().size(); i++)
{
builder.addClusteringColumn("clustering" + (i > 0 ? i : ""), header.getClusteringTypes().get(i));
}
return builder.build();
}
private static <T> Stream<T> iterToStream(Iterator<T> iter)
{
Spliterator<T> splititer = Spliterators.spliteratorUnknownSize(iter, Spliterator.IMMUTABLE);
return StreamSupport.stream(splititer, false);
}
/**
* Given arguments specifying an SSTable, and optionally an output file, export the contents of the SSTable to JSON.
*
* @param args
* command lines arguments
* @throws ConfigurationException
* on configuration failure (wrong params given)
*/
public static void main(String[] args) throws ConfigurationException
{
CommandLineParser parser = new PosixParser();
try
{
cmd = parser.parse(options, args);
}
catch (ParseException e1)
{
System.err.println(e1.getMessage());
printUsage();
System.exit(1);
}
if (cmd.getArgs().length != 1)
{
System.err.println("You must supply exactly one sstable");
printUsage();
System.exit(1);
}
String[] keys = cmd.getOptionValues(KEY_OPTION);
HashSet<String> excludes = new HashSet<>(Arrays.asList(
cmd.getOptionValues(EXCLUDE_KEY_OPTION) == null
? new String[0]
: cmd.getOptionValues(EXCLUDE_KEY_OPTION)));
String ssTableFileName = new File(cmd.getArgs()[0]).getAbsolutePath();
if (Descriptor.isLegacyFile(new File(ssTableFileName)))
{
System.err.println("Unsupported legacy sstable");
System.exit(1);
}
if (!new File(ssTableFileName).exists())
{
System.err.println("Cannot find file " + ssTableFileName);
System.exit(1);
}
Descriptor desc = Descriptor.fromFilename(ssTableFileName);
try
{
CFMetaData metadata = metadataFromSSTable(desc);
if (cmd.hasOption(ENUMERATE_KEYS_OPTION))
{
JsonTransformer.keysToJson(null, iterToStream(new KeyIterator(desc, metadata)), metadata, System.out);
}
else
{
SSTableReader sstable = SSTableReader.openNoValidation(desc, metadata);
IPartitioner partitioner = sstable.getPartitioner();
final ISSTableScanner currentScanner;
if ((keys != null) && (keys.length > 0))
{
List<AbstractBounds<PartitionPosition>> bounds = Arrays.stream(keys)
.filter(key -> !excludes.contains(key))
.map(metadata.getKeyValidator()::fromString)
.map(partitioner::decorateKey)
.sorted()
.map(DecoratedKey::getToken)
.map(token -> new Bounds<>(token.minKeyBound(), token.maxKeyBound())).collect(Collectors.toList());
currentScanner = sstable.getScanner(bounds.iterator());
}
else
{
currentScanner = sstable.getScanner();
}
Stream<UnfilteredRowIterator> partitions = iterToStream(currentScanner).filter(i ->
excludes.isEmpty() || !excludes.contains(metadata.getKeyValidator().getString(i.partitionKey().getKey()))
);
if (cmd.hasOption(DEBUG_OUTPUT_OPTION))
{
AtomicLong position = new AtomicLong();
partitions.forEach(partition ->
{
position.set(currentScanner.getCurrentPosition());
partition.forEachRemaining(row ->
{
System.out.println(
"[" + metadata.getKeyValidator().getString(partition.partitionKey().getKey()) + "]@"
+ position.get() + " " + row.toString(metadata, false, true));
position.set(currentScanner.getCurrentPosition());
});
});
}
else
{
JsonTransformer.toJson(currentScanner, partitions, metadata, System.out);
}
}
}
catch (IOException e)
{
// throwing exception outside main with broken pipe causes windows cmd to hang
e.printStackTrace(System.err);
}
System.exit(0);
}
private static void printUsage()
{
String usage = String.format("sstabledump <options> <sstable file path>%n");
String header = "Dump contents of given SSTable to standard output in JSON format.";
new HelpFormatter().printHelp(usage, header, options, "");
}
}

8
test/unit/org/apache/cassandra/io/sstable/SSTableScannerTest.java
View File

@ -24,6 +24,8 @@ import java.util.Collection;
import java.util.List;
import com.google.common.collect.Iterables;
import com.google.common.util.concurrent.RateLimiter;
import org.junit.BeforeClass;
import org.junit.Test;
@ -217,7 +219,7 @@ public class SSTableScannerTest
SSTableReader sstable = store.getLiveSSTables().iterator().next();
// full range scan
ISSTableScanner scanner = sstable.getScanner(null);
ISSTableScanner scanner = sstable.getScanner(RateLimiter.create(Double.MAX_VALUE));
for (int i = 2; i < 10; i++)
assertEquals(toKey(i), new String(scanner.next().partitionKey().getKey().array()));
@ -323,7 +325,7 @@ public class SSTableScannerTest
SSTableReader sstable = store.getLiveSSTables().iterator().next();
// full range scan
ISSTableScanner fullScanner = sstable.getScanner(null);
ISSTableScanner fullScanner = sstable.getScanner(RateLimiter.create(Double.MAX_VALUE));
assertScanContainsRanges(fullScanner,
2, 9,
102, 109,
@ -453,7 +455,7 @@ public class SSTableScannerTest
SSTableReader sstable = store.getLiveSSTables().iterator().next();
// full range scan
ISSTableScanner fullScanner = sstable.getScanner(null);
ISSTableScanner fullScanner = sstable.getScanner(RateLimiter.create(Double.MAX_VALUE));
assertScanContainsRanges(fullScanner, 205, 205);
// scan three ranges separately

52
tools/bin/sstabledump Executable file
View File

@ -0,0 +1,52 @@
#!/bin/sh
# Licensed to the Apache Software Foundation (ASF) under one
# or more contributor license agreements. See the NOTICE file
# distributed with this work for additional information
# regarding copyright ownership. The ASF licenses this file
# to you under the Apache License, Version 2.0 (the
# "License"); you may not use this file except in compliance
# with the License. You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
if [ "x$CASSANDRA_INCLUDE" = "x" ]; then
for include in "`dirname "$0"`/cassandra.in.sh" \
"$HOME/.cassandra.in.sh" \
/usr/share/cassandra/cassandra.in.sh \
/usr/local/share/cassandra/cassandra.in.sh \
/opt/cassandra/cassandra.in.sh; do
if [ -r "$include" ]; then
. "$include"
break
fi
done
elif [ -r "$CASSANDRA_INCLUDE" ]; then
. "$CASSANDRA_INCLUDE"
fi
# Use JAVA_HOME if set, otherwise look for java in PATH
if [ -x "$JAVA_HOME/bin/java" ]; then
JAVA="$JAVA_HOME/bin/java"
else
JAVA="`which java`"
fi
if [ -z "$CLASSPATH" ]; then
echo "You must set the CLASSPATH var" >&2
exit 1
fi
"$JAVA" $JAVA_AGENT -cp "$CLASSPATH" $JVM_OPTS -Dstorage-config="$CASSANDRA_CONF" \
-Dcassandra.storagedir="$cassandra_storagedir" \
-Dlogback.configurationFile=logback-tools.xml \
org.apache.cassandra.tools.SSTableExport "$@"
# vi:ai sw=4 ts=4 tw=0 et

48
tools/bin/sstabledump.bat Normal file
View File

@ -0,0 +1,48 @@
@REM
@REM Licensed to the Apache Software Foundation (ASF) under one or more
@REM contributor license agreements. See the NOTICE file distributed with
@REM this work for additional information regarding copyright ownership.
@REM The ASF licenses this file to You under the Apache License, Version 2.0
@REM (the "License"); you may not use this file except in compliance with
@REM the License. You may obtain a copy of the License at
@REM
@REM http://www.apache.org/licenses/LICENSE-2.0
@REM
@REM Unless required by applicable law or agreed to in writing, software
@REM distributed under the License is distributed on an "AS IS" BASIS,
@REM WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
@REM See the License for the specific language governing permissions and
@REM limitations under the License.
@echo off
if "%OS%" == "Windows_NT" setlocal
pushd "%~dp0"
call cassandra.in.bat
if NOT DEFINED CASSANDRA_MAIN set CASSANDRA_MAIN=org.apache.cassandra.tools.SSTableExport
if NOT DEFINED JAVA_HOME goto :err
REM ***** JAVA options *****
set JAVA_OPTS=^
-Dlogback.configurationFile=logback-tools.xml
set TOOLS_PARAMS=
FOR %%A IN (%*) DO call :appendToolsParams %%A
goto runTool
:appendToolsParams
set TOOLS_PARAMS=%TOOLS_PARAMS% %1
goto :eof
:runTool
"%JAVA_HOME%\bin\java" %JAVA_OPTS% %CASSANDRA_PARAMS% -cp %CASSANDRA_CLASSPATH% "%CASSANDRA_MAIN%" %TOOLS_PARAMS%
goto finally
:err
echo JAVA_HOME environment variable must be set!
pause
:finally
ENDLOCAL