natas_hw
/
cassandra
mirror of https://github.com/apache/cassandra
0
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

make sstable2json output more readable, and remove support for supercolumn json (CASSANDRA-7078) 

patch by jbellis; reviewed by ayeschenko for CASSANDRA-7078
This commit is contained in:
Jonathan Ellis 2014-04-23 22:00:13 -05:00
parent 6acbb33fa4
commit 647bfc6d79
12 changed files with 76 additions and 271 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

3
NEWS.txt
View File

@ -48,9 +48,10 @@ Upgrading
to change your data model to accomodate the new implementation. to change your data model to accomodate the new implementation.
(See https://issues.apache.org/jira/browse/CASSANDRA-6504 and the dev (See https://issues.apache.org/jira/browse/CASSANDRA-6504 and the dev
blog post at http://www.datastax.com/dev/blog/<PLACEHOLDER> for details). blog post at http://www.datastax.com/dev/blog/<PLACEHOLDER> for details).
- (per-table) index_interval parameter has been replaced with - (per-table) index_interval parameter has been replaced with
min_index_interval and max_index_interval paratemeters. index_interval min_index_interval and max_index_interval paratemeters. index_interval
has been deprecated. has been deprecated.
- support for supercolumns has been removed from json2sstable
2.0.7 2.0.7

93
src/java/org/apache/cassandra/tools/SSTableExport.java
View File

@ -20,7 +20,6 @@ package org.apache.cassandra.tools;
import java.io.File; import java.io.File;
import java.io.IOException; import java.io.IOException;
import java.io.PrintStream; import java.io.PrintStream;
import java.nio.ByteBuffer;
import java.util.*; import java.util.*;
import org.apache.commons.cli.*; import org.apache.commons.cli.*;
@ -98,53 +97,6 @@ public class SSTableExport
out.print(": "); out.print(": ");
} }
/**
* JSON ColumnFamily metadata serializer.</br> Serializes:
* <ul>
* <li>column family deletion info (if present)</li>
* </ul>
*
* @param out The output steam to write data
* @param deletionInfo
*/
private static void writeMeta(PrintStream out, DeletionInfo deletionInfo)
{
if (!deletionInfo.isLive())
{
// begin meta
writeKey(out, "metadata");
writeDeletionInfo(out, deletionInfo.getTopLevelDeletion());
out.print(",");
}
}
private static void writeDeletionInfo(PrintStream out, DeletionTime deletionTime)
{
out.print("{");
writeKey(out, "deletionInfo");
// only store topLevelDeletion (serializeForSSTable only uses this)
writeJSON(out, deletionTime);
out.print("}");
}
/**
* Serialize columns using given column iterator
*
* @param atoms column iterator
* @param out output stream
* @param cfMetaData Column Family metadata (to get validator)
*/
private static void serializeAtoms(Iterator<OnDiskAtom> atoms, PrintStream out, CFMetaData cfMetaData)
{
while (atoms.hasNext())
{
writeJSON(out, serializeAtom(atoms.next(), cfMetaData));
if (atoms.hasNext())
out.print(", ");
}
}
private static List<Object> serializeAtom(OnDiskAtom atom, CFMetaData cfMetaData) private static List<Object> serializeAtom(OnDiskAtom atom, CFMetaData cfMetaData)
{ {
if (atom instanceof Cell) if (atom instanceof Cell)
@ -166,7 +118,17 @@ public class SSTableExport
} }
/** /**
* Serialize a given cell to the JSON format * Serialize a given cell to a List of Objects that jsonMapper knows how to turn into strings. Format is
*
* human_readable_name, value, timestamp, [flag, [options]]
*
* Value is normally the human readable value as rendered by the validator, but for deleted cells we
* give the local deletion time instead.
*
* Flag may be exactly one of {d,e,c} for deleted, expiring, or counter:
* - No options for deleted cells
* - If expiring, options will include the TTL and local deletion time.
* - If counter, options will include timestamp of last delete
* *
* @param cell cell presentation * @param cell cell presentation
* @param cfMetaData Column Family metadata (to get validator) * @param cfMetaData Column Family metadata (to get validator)
@ -177,18 +139,18 @@ public class SSTableExport
CellNameType comparator = cfMetaData.comparator; CellNameType comparator = cfMetaData.comparator;
ArrayList<Object> serializedColumn = new ArrayList<Object>(); ArrayList<Object> serializedColumn = new ArrayList<Object>();
ByteBuffer value = cell.value();
serializedColumn.add(comparator.getString(cell.name())); serializedColumn.add(comparator.getString(cell.name()));
if (cell instanceof DeletedCell) if (cell instanceof DeletedCell)
{ {
serializedColumn.add(ByteBufferUtil.bytesToHex(value)); serializedColumn.add(cell.getLocalDeletionTime());
} }
else else
{ {
AbstractType<?> validator = cfMetaData.getValueValidator(cell.name()); AbstractType<?> validator = cfMetaData.getValueValidator(cell.name());
serializedColumn.add(validator.getString(value)); serializedColumn.add(validator.getString(cell.value()));
} }
serializedColumn.add(cell.timestamp()); serializedColumn.add(cell.timestamp());
if (cell instanceof DeletedCell) if (cell instanceof DeletedCell)
@ -227,16 +189,31 @@ public class SSTableExport
out.print("{"); out.print("{");
writeKey(out, "key"); writeKey(out, "key");
writeJSON(out, bytesToHex(key.key)); writeJSON(out, bytesToHex(key.key));
out.print(","); out.print(",\n");
writeMeta(out, deletionInfo); if (!deletionInfo.isLive())
{
out.print(" ");
writeKey(out, "metadata");
out.print("{");
writeKey(out, "deletionInfo");
writeJSON(out, deletionInfo.getTopLevelDeletion());
out.print("}");
out.print(",\n");
}
writeKey(out, "columns"); out.print(" ");
writeKey(out, "cells");
out.print("["); out.print("[");
while (atoms.hasNext())
{
writeJSON(out, serializeAtom(atoms.next(), metadata));
serializeAtoms(atoms, out, metadata); if (atoms.hasNext())
out.print(",\n ");
}
out.print("]"); out.print("]");
out.print("}"); out.print("}");
} }

133
src/java/org/apache/cassandra/tools/SSTableImport.java
View File

@ -63,14 +63,12 @@ public class SSTableImport
private static final String COLUMN_FAMILY_OPTION = "c"; private static final String COLUMN_FAMILY_OPTION = "c";
private static final String KEY_COUNT_OPTION = "n"; private static final String KEY_COUNT_OPTION = "n";
private static final String IS_SORTED_OPTION = "s"; private static final String IS_SORTED_OPTION = "s";
private static final String OLD_SC_FORMAT_OPTION = "S";
private static final Options options = new Options(); private static final Options options = new Options();
private static CommandLine cmd; private static CommandLine cmd;
private Integer keyCountToImport; private Integer keyCountToImport;
private final boolean isSorted; private final boolean isSorted;
private final boolean oldSCFormat;
private static final JsonFactory factory = new MappingJsonFactory().configure( private static final JsonFactory factory = new MappingJsonFactory().configure(
JsonParser.Feature.INTERN_FIELD_NAMES, false); JsonParser.Feature.INTERN_FIELD_NAMES, false);
@ -87,7 +85,6 @@ public class SSTableImport
options.addOption(new Option(KEY_COUNT_OPTION, true, "Number of keys to import (Optional).")); options.addOption(new Option(KEY_COUNT_OPTION, true, "Number of keys to import (Optional)."));
options.addOption(new Option(IS_SORTED_OPTION, false, "Assume JSON file as already sorted (e.g. created by sstable2json tool) (Optional).")); options.addOption(new Option(IS_SORTED_OPTION, false, "Assume JSON file as already sorted (e.g. created by sstable2json tool) (Optional)."));
options.addOption(new Option(OLD_SC_FORMAT_OPTION, false, "Assume JSON file use legacy super column format (Optional)."));
} }
private static class JsonColumn<T> private static class JsonColumn<T>
@ -104,11 +101,11 @@ public class SSTableImport
// Counter columns // Counter columns
private long timestampOfLastDelete; private long timestampOfLastDelete;
public JsonColumn(T json, CFMetaData meta, boolean oldSCFormat, boolean isSubColumn) public JsonColumn(T json, CFMetaData meta)
{ {
if (json instanceof List) if (json instanceof List)
{ {
CellNameType comparator = oldSCFormat ? new SimpleDenseCellNameType(SuperColumns.getComparatorFor(meta, isSubColumn)) : meta.comparator; CellNameType comparator = meta.comparator;
List fields = (List<?>) json; List fields = (List<?>) json;
assert fields.size() >= 3 : "Cell definition should have at least 3"; assert fields.size() >= 3 : "Cell definition should have at least 3";
@ -119,42 +116,25 @@ public class SSTableImport
if (fields.size() > 3) if (fields.size() > 3)
{ {
if (fields.get(3) instanceof Boolean) kind = (String) fields.get(3);
if (isExpiring())
{ {
// old format, reading this for backward compatibility sake ttl = (Integer) fields.get(4);
if (fields.size() == 6) localExpirationTime = (Integer) fields.get(5);
{
kind = "e";
ttl = (Integer) fields.get(4);
localExpirationTime = (Integer) fields.get(5);
}
else
{
kind = ((Boolean) fields.get(3)) ? "d" : "";
}
} }
else else if (isCounter())
{ {
kind = (String) fields.get(3); timestampOfLastDelete = (long) ((Integer) fields.get(4));
if (isExpiring()) }
{ else if (isRangeTombstone())
ttl = (Integer) fields.get(4); {
localExpirationTime = (Integer) fields.get(5); localExpirationTime = (Integer) fields.get(4);
}
else if (isCounter())
{
timestampOfLastDelete = (long) ((Integer) fields.get(4));
}
else if (isRangeTombstone())
{
localExpirationTime = (Integer) fields.get(4);
}
} }
} }
if (isDeleted()) if (isDeleted())
{ {
value = ByteBufferUtil.hexToBytes((String) fields.get(1)); value = ByteBufferUtil.bytes((Integer) fields.get(1));
} }
else if (isRangeTombstone()) else if (isRangeTombstone())
{ {
@ -200,61 +180,43 @@ public class SSTableImport
public SSTableImport() public SSTableImport()
{ {
this(null, false, false); this(null, false);
} }
public SSTableImport(boolean isSorted) public SSTableImport(boolean isSorted)
{ {
this(isSorted, false); this(null, isSorted);
} }
public SSTableImport(boolean isSorted, boolean oldSCFormat) public SSTableImport(Integer keyCountToImport, boolean isSorted)
{
this(null, isSorted, oldSCFormat);
}
public SSTableImport(Integer keyCountToImport, boolean isSorted, boolean oldSCFormat)
{ {
this.keyCountToImport = keyCountToImport; this.keyCountToImport = keyCountToImport;
this.isSorted = isSorted; this.isSorted = isSorted;
this.oldSCFormat = oldSCFormat;
}
private void addToStandardCF(List<?> row, ColumnFamily cfamily)
{
addColumnsToCF(row, null, cfamily);
} }
/** /**
* Add columns to a column family. * Add columns to a column family.
* *
* @param row the columns associated with a row * @param row the columns associated with a row
* @param superName name of the super column if any
* @param cfamily the column family to add columns to * @param cfamily the column family to add columns to
*/ */
private void addColumnsToCF(List<?> row, ByteBuffer superName, ColumnFamily cfamily) private void addColumnsToCF(List<?> row, ColumnFamily cfamily)
{ {
CFMetaData cfm = cfamily.metadata(); CFMetaData cfm = cfamily.metadata();
assert cfm != null; assert cfm != null;
for (Object c : row) for (Object c : row)
{ {
JsonColumn col = new JsonColumn<List>((List) c, cfm, oldSCFormat, (superName != null)); JsonColumn col = new JsonColumn<List>((List) c, cfm);
if (col.isRangeTombstone()) if (col.isRangeTombstone())
{ {
Composite start = superName == null Composite start = cfm.comparator.fromByteBuffer(col.getName());
? cfm.comparator.fromByteBuffer(col.getName()) Composite end = cfm.comparator.fromByteBuffer(col.getValue());
: cfm.comparator.make(superName, col.getName());
Composite end = superName == null
? cfm.comparator.fromByteBuffer(col.getValue())
: cfm.comparator.make(superName, col.getValue());
cfamily.addAtom(new RangeTombstone(start, end, col.timestamp, col.localExpirationTime)); cfamily.addAtom(new RangeTombstone(start, end, col.timestamp, col.localExpirationTime));
continue; continue;
} }
CellName cname = superName == null CellName cname = cfm.comparator.cellFromByteBuffer(col.getName());
? cfm.comparator.cellFromByteBuffer(col.getName())
: cfm.comparator.makeCellName(superName, col.getName());
if (col.isExpiring()) if (col.isExpiring())
{ {
@ -270,9 +232,7 @@ public class SSTableImport
} }
else if (col.isRangeTombstone()) else if (col.isRangeTombstone())
{ {
CellName end = superName == null CellName end = cfm.comparator.cellFromByteBuffer(col.getValue());
? cfm.comparator.cellFromByteBuffer(col.getValue())
: cfm.comparator.makeCellName(superName, col.getValue());
cfamily.addAtom(new RangeTombstone(cname, end, col.timestamp, col.localExpirationTime)); cfamily.addAtom(new RangeTombstone(cname, end, col.timestamp, col.localExpirationTime));
} }
// cql3 row marker, see CASSANDRA-5852 // cql3 row marker, see CASSANDRA-5852
@ -304,35 +264,6 @@ public class SSTableImport
} }
} }
/**
* Add super columns to a column family.
*
* @param row the super columns associated with a row
* @param cfamily the column family to add columns to
*/
private void addToSuperCF(Map<?, ?> row, ColumnFamily cfamily)
{
CFMetaData metaData = cfamily.metadata();
assert metaData != null;
CellNameType comparator = metaData.comparator;
// Super columns
for (Map.Entry<?, ?> entry : row.entrySet())
{
Map<?, ?> data = (Map<?, ?>) entry.getValue();
ByteBuffer superName = stringAsType((String) entry.getKey(), comparator.subtype(0));
addColumnsToCF((List<?>) data.get("subColumns"), superName, cfamily);
if (data.containsKey("metadata"))
{
parseMeta((Map<?, ?>) data.get("metadata"), cfamily, superName);
}
}
}
/** /**
* Convert a JSON formatted file to an SSTable. * Convert a JSON formatted file to an SSTable.
* *
@ -387,11 +318,8 @@ public class SSTableImport
parseMeta((Map<?, ?>) row.getValue().get("metadata"), columnFamily, null); parseMeta((Map<?, ?>) row.getValue().get("metadata"), columnFamily, null);
} }
Object columns = row.getValue().get("columns"); Object columns = row.getValue().get("cells");
if (columnFamily.getType() == ColumnFamilyType.Super && oldSCFormat) addColumnsToCF((List<?>) columns, columnFamily);
addToSuperCF((Map<?, ?>) columns, columnFamily);
else
addToStandardCF((List<?>) columns, columnFamily);
writer.append(row.getKey(), columnFamily); writer.append(row.getKey(), columnFamily);
@ -458,10 +386,7 @@ public class SSTableImport
if (row.containsKey("metadata")) if (row.containsKey("metadata"))
parseMeta((Map<?, ?>) row.get("metadata"), columnFamily, null); parseMeta((Map<?, ?>) row.get("metadata"), columnFamily, null);
if (columnFamily.getType() == ColumnFamilyType.Super && oldSCFormat) addColumnsToCF((List<?>) row.get("cells"), columnFamily);
addToSuperCF((Map<?, ?>)row.get("columns"), columnFamily);
else
addToStandardCF((List<?>)row.get("columns"), columnFamily);
if (prevStoredKey != null && prevStoredKey.compareTo(currentKey) != -1) if (prevStoredKey != null && prevStoredKey.compareTo(currentKey) != -1)
{ {
@ -545,7 +470,6 @@ public class SSTableImport
Integer keyCountToImport = null; Integer keyCountToImport = null;
boolean isSorted = false; boolean isSorted = false;
boolean oldSCFormat = false;
if (cmd.hasOption(KEY_COUNT_OPTION)) if (cmd.hasOption(KEY_COUNT_OPTION))
{ {
@ -557,11 +481,6 @@ public class SSTableImport
isSorted = true; isSorted = true;
} }
if (cmd.hasOption(OLD_SC_FORMAT_OPTION))
{
oldSCFormat = true;
}
DatabaseDescriptor.loadSchemas(); DatabaseDescriptor.loadSchemas();
if (Schema.instance.getNonSystemKeyspaces().size() < 1) if (Schema.instance.getNonSystemKeyspaces().size() < 1)
{ {
@ -572,7 +491,7 @@ public class SSTableImport
try try
{ {
new SSTableImport(keyCountToImport, isSorted, oldSCFormat).importJson(json, keyspace, cfamily, ssTable); new SSTableImport(keyCountToImport, isSorted).importJson(json, keyspace, cfamily, ssTable);
} }
catch (Exception e) catch (Exception e)
{ {

2
test/resources/CounterCF.json
View File

@ -1,3 +1,3 @@
[ [
{"key": "726f7741", "columns": [["636f6c4141", "000100008c619170467411e00000fe8ebeead9ee0000000000000001000000000000002a", 1294532915068, "c", 0]]} {"key": "726f7741", "cells": [["636f6c4141", "000100008c619170467411e00000fe8ebeead9ee0000000000000001000000000000002a", 1294532915068, "c", 0]]}
] ]

4
test/resources/SimpleCF.json
View File

@ -1,4 +1,4 @@
[ [
{"key": "726f7741", "columns": [["636f6c4141", "76616c4141", 1294532915068], ["636f6c4142", "76616c4142", 1294532915069], ["636f6c4143", "76616c4143", 1294532915071, "e", 42, 2000000000]]}, {"key": "726f7741", "cells": [["636f6c4141", "76616c4141", 1294532915068], ["636f6c4142", "76616c4142", 1294532915069], ["636f6c4143", "76616c4143", 1294532915071, "e", 42, 2000000000]]},
{"key": "726f7742", "columns": [["636f6c4241", "76616c4241", 1294532915070], ["636f6c4242", "76616c4242", 1294532915073]]} {"key": "726f7742", "cells": [["636f6c4241", "76616c4241", 1294532915070], ["636f6c4242", "76616c4242", 1294532915073]]}
] ]

4
test/resources/SimpleCF.oldformat.json
View File

@ -1,4 +0,0 @@
[
{"key": "726f7741", "columns": [["636f6c4141", "76616c4141", 1294532915068, false], ["636f6c4142", "76616c4142", 1294532915069, false], ["636f6c4143", "76616c4143", 1294532915071, false, 42, 2000000000 ]]},
{"key": "726f7742", "columns": [["636f6c4241", "76616c4241", 1294532915070, false], ["636f6c4242", "76616c4242", 1294532915073, false]]}
]

4
test/resources/SimpleCFWithDeletionInfo.json
View File

@ -1,4 +1,4 @@
[ [
{"key": "726f7741","metadata":{"deletionInfo":{"markedForDeleteAt":0,"localDeletionTime":0}}, "columns": [["636f6c4141", "76616c4141", 1294532915068], ["636f6c4142", "76616c4142", 1294532915069], ["636f6c4143", "76616c4143", 1294532915071, "e", 42, 2000000000]]}, {"key": "726f7741","metadata":{"deletionInfo":{"markedForDeleteAt":0,"localDeletionTime":0}}, "cells": [["636f6c4141", "76616c4141", 1294532915068], ["636f6c4142", "76616c4142", 1294532915069], ["636f6c4143", "76616c4143", 1294532915071, "e", 42, 2000000000]]},
{"key": "726f7742","metadata":{"deletionInfo":{"markedForDeleteAt":0,"localDeletionTime":0}}, "columns": [["636f6c4241", "76616c4241", 1294532915070], ["636f6c4242", "76616c4242", 1294532915073]]} {"key": "726f7742","metadata":{"deletionInfo":{"markedForDeleteAt":0,"localDeletionTime":0}}, "cells": [["636f6c4241", "76616c4241", 1294532915070], ["636f6c4242", "76616c4242", 1294532915073]]}
] ]

4
test/resources/SuperCF.json
View File

@ -1,4 +0,0 @@
[
{"key": "726f7741", "columns": {"737570657241": {"metadata": {"deletionInfo": {"markedForDeleteAt":0,"localDeletionTime":0}}, "subColumns": [["636f6c4141", "76616c75654141", 1294532915069], ["636f6c4142", "76616c75654142", 1294532915069]]}}},
{"key": "726f7742", "columns": {"737570657242": {"subColumns": [["636f6c4241", "76616c75654241", 1294532915069], ["636f6c4242", "76616c75654242", 1294532915069]]}}}
]

4
test/resources/UnsortedCF.json
View File

@ -1,4 +1,4 @@
[ [
{"key": "726f7742", "columns": [["636f6c4241", "76616c4241", 1294532915070], ["636f6c4242", "76616c4242", 1294532915073]]}, {"key": "726f7742", "cells": [["636f6c4241", "76616c4241", 1294532915070], ["636f6c4242", "76616c4242", 1294532915073]]},
{"key": "726f7741", "columns": [["636f6c4141", "76616c4141", 1294532915068], ["636f6c4142", "76616c4142", 1294532915069], ["636f6c4143", "76616c4143", 1294532915071, "e", 42, 2000000000]]} {"key": "726f7741", "cells": [["636f6c4141", "76616c4141", 1294532915068], ["636f6c4142", "76616c4142", 1294532915069], ["636f6c4143", "76616c4143", 1294532915071, "e", 42, 2000000000]]}
] ]

5
test/resources/UnsortedSuperCF.json
View File

@ -1,5 +0,0 @@
[
{"key": "303935", "columns": { "5330": {"deletedAt": -9223372036854775808, "subColumns": [["4330", "366338333439636337323630", 1294656637116, false], ["4331", "366338333439636337323630", 1294656637116, false], ["4332", "366338333439636337323630", 1294656637116, false], ["4333", "366338333439636337323630", 1294656637116, false], ["4334", "366338333439636337323630", 1294656637116, false]]}}} ,
{"key": "303630", "columns": { "5330": {"deletedAt": -9223372036854775808, "subColumns": [["4330", "643364393434363830326134", 1294656636902, false], ["4331", "643364393434363830326134", 1294656636902, false], ["4332", "643364393434363830326134", 1294656636902, false], ["4333", "643364393434363830326134", 1294656636902, false], ["4334", "643364393434363830326134", 1294656636902, false]]}}} ,
{"key": "303638", "columns": { "5330": {"deletedAt": -9223372036854775808, "subColumns": [["4330", "366634393232663435353638", 1294656636885, false], ["4331", "366634393232663435353638", 1294656636885, false], ["4332", "366634393232663435353638", 1294656636885, false], ["4333", "366634393232663435353638", 1294656636885, false], ["4334", "366634393232663435353638", 1294656636885, false]]}}}
]

40
test/unit/org/apache/cassandra/tools/SSTableExportTest.java
View File

@ -56,31 +56,6 @@ public class SSTableExportTest extends SchemaLoader
return bytesToHex(ByteBufferUtil.bytes(str)); return bytesToHex(ByteBufferUtil.bytes(str));
} }
public SSTableWriter getDummyWriter() throws IOException
{
File tempSS = tempSSTableFile("Keyspace1", "Standard1");
ColumnFamily cfamily = ArrayBackedSortedColumns.factory.create("Keyspace1", "Standard1");
SSTableWriter writer = new SSTableWriter(tempSS.getPath(), 2, ActiveRepairService.UNREPAIRED_SSTABLE);
// Add rowA
cfamily.addColumn(Util.cellname("colA"), ByteBufferUtil.bytes("valA"), System.currentTimeMillis());
writer.append(Util.dk("rowA"), cfamily);
cfamily.clear();
cfamily.addColumn(Util.cellname("colB"), ByteBufferUtil.bytes("valB"), System.currentTimeMillis());
writer.append(Util.dk("rowB"), cfamily);
cfamily.clear();
return writer;
}
public PrintStream dummyStream = new PrintStream(new OutputStream(){
public void write(int b) throws IOException { throw new IOException(); }
});
@Test @Test
public void testEnumeratekeys() throws IOException public void testEnumeratekeys() throws IOException
{ {
@ -153,7 +128,7 @@ public class SSTableExportTest extends SchemaLoader
assertEquals("unexpected number of keys", 2, rowA.keySet().size()); assertEquals("unexpected number of keys", 2, rowA.keySet().size());
assertEquals("unexpected row key",asHex("rowA"),rowA.get("key")); assertEquals("unexpected row key",asHex("rowA"),rowA.get("key"));
JSONArray colsA = (JSONArray)rowA.get("columns"); JSONArray colsA = (JSONArray)rowA.get("cells");
JSONArray colA = (JSONArray)colsA.get(0); JSONArray colA = (JSONArray)colsA.get(0);
assert hexToBytes((String)colA.get(1)).equals(ByteBufferUtil.bytes("valA")); assert hexToBytes((String)colA.get(1)).equals(ByteBufferUtil.bytes("valA"));
@ -165,7 +140,7 @@ public class SSTableExportTest extends SchemaLoader
assertEquals("unexpected number of keys", 2, rowB.keySet().size()); assertEquals("unexpected number of keys", 2, rowB.keySet().size());
assertEquals("unexpected row key",asHex("rowB"),rowB.get("key")); assertEquals("unexpected row key",asHex("rowB"),rowB.get("key"));
JSONArray colsB = (JSONArray)rowB.get("columns"); JSONArray colsB = (JSONArray)rowB.get("cells");
JSONArray colB = (JSONArray)colsB.get(0); JSONArray colB = (JSONArray)colsB.get(0);
assert colB.size() == 3; assert colB.size() == 3;
@ -235,7 +210,7 @@ public class SSTableExportTest extends SchemaLoader
assertEquals("unexpected number of keys", 2, row.keySet().size()); assertEquals("unexpected number of keys", 2, row.keySet().size());
assertEquals("unexpected row key",asHex("rowA"),row.get("key")); assertEquals("unexpected row key",asHex("rowA"),row.get("key"));
JSONArray cols = (JSONArray)row.get("columns"); JSONArray cols = (JSONArray)row.get("cells");
JSONArray colA = (JSONArray)cols.get(0); JSONArray colA = (JSONArray)cols.get(0);
assert hexToBytes((String)colA.get(0)).equals(ByteBufferUtil.bytes("colA")); assert hexToBytes((String)colA.get(0)).equals(ByteBufferUtil.bytes("colA"));
assert ((String) colA.get(3)).equals("c"); assert ((String) colA.get(3)).equals("c");
@ -267,7 +242,7 @@ public class SSTableExportTest extends SchemaLoader
assertEquals("unexpected number of keys", 2, row.keySet().size()); assertEquals("unexpected number of keys", 2, row.keySet().size());
assertEquals("unexpected row key",asHex("rowA"),row.get("key")); assertEquals("unexpected row key",asHex("rowA"),row.get("key"));
JSONArray cols = (JSONArray)row.get("columns"); JSONArray cols = (JSONArray)row.get("cells");
JSONArray colA = (JSONArray)cols.get(0); JSONArray colA = (JSONArray)cols.get(0);
assert hexToBytes((String)colA.get(0)).equals(ByteBufferUtil.bytes("data")); assert hexToBytes((String)colA.get(0)).equals(ByteBufferUtil.bytes("data"));
assert colA.get(1).equals("{\"foo\":\"bar\"}"); assert colA.get(1).equals("{\"foo\":\"bar\"}");
@ -276,7 +251,6 @@ public class SSTableExportTest extends SchemaLoader
@Test @Test
public void testExportColumnsWithMetadata() throws IOException, ParseException public void testExportColumnsWithMetadata() throws IOException, ParseException
{ {
File tempSS = tempSSTableFile("Keyspace1", "Standard1"); File tempSS = tempSSTableFile("Keyspace1", "Standard1");
ColumnFamily cfamily = ArrayBackedSortedColumns.factory.create("Keyspace1", "Standard1"); ColumnFamily cfamily = ArrayBackedSortedColumns.factory.create("Keyspace1", "Standard1");
SSTableWriter writer = new SSTableWriter(tempSS.getPath(), 2, ActiveRepairService.UNREPAIRED_SSTABLE); SSTableWriter writer = new SSTableWriter(tempSS.getPath(), 2, ActiveRepairService.UNREPAIRED_SSTABLE);
@ -293,7 +267,6 @@ public class SSTableExportTest extends SchemaLoader
SSTableExport.export(reader, new PrintStream(tempJson.getPath()), new String[0]); SSTableExport.export(reader, new PrintStream(tempJson.getPath()), new String[0]);
JSONArray json = (JSONArray)JSONValue.parseWithException(new FileReader(tempJson)); JSONArray json = (JSONArray)JSONValue.parseWithException(new FileReader(tempJson));
System.out.println(json.toJSONString());
assertEquals("unexpected number of rows", 1, json.size()); assertEquals("unexpected number of rows", 1, json.size());
JSONObject row = (JSONObject)json.get(0); JSONObject row = (JSONObject)json.get(0);
@ -320,7 +293,7 @@ public class SSTableExportTest extends SchemaLoader
serializedDeletionInfo.toJSONString()); serializedDeletionInfo.toJSONString());
// check the colums are what we put in // check the colums are what we put in
JSONArray cols = (JSONArray) row.get("columns"); JSONArray cols = (JSONArray) row.get("cells");
assertNotNull("expecing columns to be present", cols); assertNotNull("expecing columns to be present", cols);
assertEquals("expecting two columns", 2, cols.size()); assertEquals("expecting two columns", 2, cols.size());
@ -331,7 +304,6 @@ public class SSTableExportTest extends SchemaLoader
JSONArray col2 = (JSONArray) cols.get(1); JSONArray col2 = (JSONArray) cols.get(1);
assertEquals("column name did not match", ByteBufferUtil.bytes("colName1"), hexToBytes((String) col2.get(0))); assertEquals("column name did not match", ByteBufferUtil.bytes("colName1"), hexToBytes((String) col2.get(0)));
assertEquals("column value did not match", ByteBufferUtil.bytes("val1"), hexToBytes((String) col2.get(1))); assertEquals("column value did not match", ByteBufferUtil.bytes("val1"), hexToBytes((String) col2.get(1)));
} }
/** /**
@ -357,7 +329,7 @@ public class SSTableExportTest extends SchemaLoader
assertEquals(1, json.size()); assertEquals(1, json.size());
JSONObject row = (JSONObject)json.get(0); JSONObject row = (JSONObject)json.get(0);
JSONArray cols = (JSONArray) row.get("columns"); JSONArray cols = (JSONArray) row.get("cells");
assertEquals(1, cols.size()); assertEquals(1, cols.size());
// check column name and value // check column name and value

51
test/unit/org/apache/cassandra/tools/SSTableImportTest.java
View File

@ -74,57 +74,6 @@ public class SSTableImportTest extends SchemaLoader
return new URI(getClass().getClassLoader().getResource(name).toString()).getPath(); return new URI(getClass().getClassLoader().getResource(name).toString()).getPath();
} }
@Test
public void testImportSimpleCfOldFormat() throws IOException, URISyntaxException
{
// Import JSON to temp SSTable file
String jsonUrl = resourcePath("SimpleCF.oldformat.json");
File tempSS = tempSSTableFile("Keyspace1", "Standard1");
new SSTableImport(true).importJson(jsonUrl, "Keyspace1", "Standard1", tempSS.getPath());
// Verify results
SSTableReader reader = SSTableReader.open(Descriptor.fromFilename(tempSS.getPath()));
QueryFilter qf = QueryFilter.getIdentityFilter(Util.dk("rowA"), "Standard1", System.currentTimeMillis());
OnDiskAtomIterator iter = qf.getSSTableColumnIterator(reader);
ColumnFamily cf = cloneForAdditions(iter);
while (iter.hasNext()) cf.addAtom(iter.next());
assert cf.getColumn(Util.cellname("colAA")).value().equals(hexToBytes("76616c4141"));
assert !(cf.getColumn(Util.cellname("colAA")) instanceof DeletedCell);
Cell expCol = cf.getColumn(Util.cellname("colAC"));
assert expCol.value().equals(hexToBytes("76616c4143"));
assert expCol instanceof ExpiringCell;
assert ((ExpiringCell)expCol).getTimeToLive() == 42 && expCol.getLocalDeletionTime() == 2000000000;
}
@Test
public void testImportSuperCf() throws IOException, URISyntaxException
{
String jsonUrl = resourcePath("SuperCF.json");
File tempSS = tempSSTableFile("Keyspace1", "Super4");
new SSTableImport(true, true).importJson(jsonUrl, "Keyspace1", "Super4", tempSS.getPath());
// Verify results
SSTableReader reader = SSTableReader.open(Descriptor.fromFilename(tempSS.getPath()));
QueryFilter qf = QueryFilter.getIdentityFilter(Util.dk("rowA"), "Super4", System.currentTimeMillis());
ColumnFamily cf = cloneForAdditions(qf.getSSTableColumnIterator(reader));
qf.collateOnDiskAtom(cf, qf.getSSTableColumnIterator(reader), Integer.MIN_VALUE);
DeletionTime delTime = cf.deletionInfo().deletionTimeFor(cf.getComparator().make(ByteBufferUtil.bytes("superA")));
assertEquals("supercolumn deletion time did not match the expected time", new DeletionInfo(0, 0), new DeletionInfo(delTime));
Cell subCell = cf.getColumn(Util.cellname("superA", "636f6c4141"));
assert subCell.value().equals(hexToBytes("76616c75654141"));
}
@Test
public void testImportUnsortedDataWithSortedOptionFails() throws IOException, URISyntaxException
{
String jsonUrl = resourcePath("UnsortedSuperCF.json");
File tempSS = tempSSTableFile("Keyspace1", "Super4");
int result = new SSTableImport(3,true, true).importJson(jsonUrl, "Keyspace1","Super4", tempSS.getPath());
assert result == -1;
}
@Test @Test
public void testImportUnsortedMode() throws IOException, URISyntaxException public void testImportUnsortedMode() throws IOException, URISyntaxException
{ {