natas_hw
/
cassandra
mirror of https://github.com/apache/cassandra
0
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

CASSANDRA-499 JSON to SSTable converter (importer) 

git-svn-id: https://svn.apache.org/repos/asf/incubator/cassandra/trunk@885322 13f79535-47bb-0310-9956-ffa450edef68
This commit is contained in:
Eric Evans 2009-11-30 03:24:46 +00:00
parent e0f5809386
commit f92915ffd5
2 changed files with 235 additions and 0 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

42
bin/json2sstable Executable file
View File

@ -0,0 +1,42 @@
#!/bin/sh
# Licensed to the Apache Software Foundation (ASF) under one
# or more contributor license agreements. See the NOTICE file
# distributed with this work for additional information
# regarding copyright ownership. The ASF licenses this file
# to you under the Apache License, Version 2.0 (the
# "License"); you may not use this file except in compliance
# with the License. You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
if [ "x$CASSANDRA_INCLUDE" = "x" ]; then
for include in /usr/share/cassandra/cassandra.in.sh \
/usr/local/share/cassandra/cassandra.in.sh \
/opt/cassandra/cassandra.in.sh \
~/.cassandra.in.sh \
`dirname $0`/cassandra.in.sh; do
if [ -r $include ]; then
. $include
break
fi
done
elif [ -r $CASSANDRA_INCLUDE ]; then
. $CASSANDRA_INCLUDE
fi
if [ -z $CLASSPATH ]; then
echo "You must set the CLASSPATH var" >&2
exit 1
fi
java -cp $CLASSPATH -Dstorage-config=$CASSANDRA_CONF \
org.apache.cassandra.tools.SSTableImport "$@"
# vi:ai sw=4 ts=4 tw=0 et

193
src/java/org/apache/cassandra/tools/SSTableImport.java Normal file
View File

@ -0,0 +1,193 @@
/**
* Licensed to the Apache Software Foundation (ASF) under one
* or more contributor license agreements. See the NOTICE file
* distributed with this work for additional information
* regarding copyright ownership. The ASF licenses this file
* to you under the Apache License, Version 2.0 (the
* "License"); you may not use this file except in compliance
* with the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.cassandra.tools;
import java.io.FileReader;
import java.io.IOException;
import java.util.*;
import org.apache.cassandra.config.DatabaseDescriptor;
import org.apache.cassandra.db.ColumnFamily;
import org.apache.cassandra.db.DecoratedKey;
import org.apache.cassandra.db.filter.QueryPath;
import org.apache.cassandra.dht.IPartitioner;
import org.apache.cassandra.io.DataOutputBuffer;
import org.apache.cassandra.io.SSTableWriter;
import org.apache.cassandra.utils.FBUtilities;
import org.apache.commons.cli.CommandLine;
import org.apache.commons.cli.CommandLineParser;
import org.apache.commons.cli.Option;
import org.apache.commons.cli.Options;
import org.apache.commons.cli.PosixParser;
import org.json.simple.JSONObject;
import org.json.simple.JSONValue;
import org.json.simple.parser.ParseException;
/**
* Create SSTables from JSON input
*/
public class SSTableImport
{
private static final String KEYSPACE_OPTION = "K";
private static final String COLFAM_OPTION = "c";
private static Options options;
private static CommandLine cmd;
static
{
options = new Options();
Option optKeyspace = new Option(KEYSPACE_OPTION, true, "Keyspace name");
optKeyspace.setRequired(true);
options.addOption(optKeyspace);
Option optColfamily = new Option(COLFAM_OPTION, true, "Column family");
optColfamily.setRequired(true);
options.addOption(optColfamily);
}
/**
* Add columns to a column family.
*
* @param row the columns associated with a row
* @param cfamily the column family to add columns to
*/
private static void addToStandardCF(JSONObject row, ColumnFamily cfamily)
{
for (Map.Entry<String, String> col : (Set<Map.Entry<String, String>>) row.entrySet())
{
QueryPath path = new QueryPath(cfamily.name(), null, col.getKey().getBytes());
byte[] value = FBUtilities.hexToBytes(col.getValue());
cfamily.addColumn(path, value, System.currentTimeMillis());
}
}
/**
* Add super columns to a column family.
*
* @param row the super columns associated with a row
* @param cfamily the column family to add columns to
*/
private static void addToSuperCF(JSONObject row, ColumnFamily cfamily)
{
// Super columns
for (Map.Entry<String, JSONObject> entry : (Set<Map.Entry<String, JSONObject>>)row.entrySet())
{
byte[] superName = entry.getKey().getBytes();
// Sub-columns
for (Map.Entry<String, String> col : (Set<Map.Entry<String, String>>)entry.getValue().entrySet())
{
QueryPath path = new QueryPath(cfamily.name(), superName, col.getKey().getBytes());
byte[] value = FBUtilities.hexToBytes(col.getValue());
cfamily.addColumn(path, value, System.currentTimeMillis());
}
}
}
/**
* Convert a JSON formatted file to an SSTable.
*
* @param jsonFile the file containing JSON formatted data
* @param keyspace keyspace the data belongs to
* @param cf column family the data belongs to
* @param ssTablePath file to write the SSTable to
* @throws IOException for errors reading/writing input/output
* @throws ParseException for errors encountered parsing JSON input
*/
private static void importJson(String jsonFile, String keyspace, String cf, String ssTablePath)
throws IOException, ParseException
{
ColumnFamily cfamily = ColumnFamily.create(keyspace, cf);
String cfType = cfamily.type(); // Super or Standard
IPartitioner<?> partitioner = DatabaseDescriptor.getPartitioner();
DataOutputBuffer dob = new DataOutputBuffer();
try
{
JSONObject json = (JSONObject)JSONValue.parse(new FileReader(jsonFile));
SSTableWriter writer = new SSTableWriter(ssTablePath, json.size(), partitioner);
List<DecoratedKey<?>> decoratedKeys = new ArrayList<DecoratedKey<?>>();
for (String key : (Set<String>)json.keySet())
decoratedKeys.add(partitioner.decorateKey(key));
Collections.sort(decoratedKeys);
for (DecoratedKey<?> rowKey : decoratedKeys)
{
JSONObject value = (JSONObject)json.get(rowKey.key);
if (cfType.equals("Super"))
addToSuperCF(value, cfamily);
else
addToStandardCF(value, cfamily);
ColumnFamily.serializer().serializeWithIndexes(cfamily, dob);
writer.append(rowKey, dob);
dob.reset();
cfamily.clear();
}
writer.closeAndOpenReader(0);
}
catch (ClassCastException cce)
{
//throw cce;
throw new RuntimeException("Invalid JSON input, or incorrect column family");
}
}
/**
* Converts JSON to an SSTable file. JSON input can either be a file specified
* using an optional command line argument, or supplied on standard in.
*
* @param args command line arguments
* @throws IOException on failure to open/read/write files or output streams
* @throws ParseException on failure to parse JSON input
*/
public static void main(String[] args) throws IOException, ParseException
{
String usage = String.format("Usage: %s -K keyspace -c column_family <json> <sstable>%n",
SSTableImport.class.getName());
CommandLineParser parser = new PosixParser();
try
{
cmd = parser.parse(options, args);
} catch (org.apache.commons.cli.ParseException e1)
{
System.err.println(e1.getMessage());
System.err.println(usage);
System.exit(1);
}
if (cmd.getArgs().length != 2)
{
System.err.println(usage);
System.exit(1);
}
String json = cmd.getArgs()[0];
String ssTable = cmd.getArgs()[1];
String keyspace = cmd.getOptionValue(KEYSPACE_OPTION);
String cfamily = cmd.getOptionValue(COLFAM_OPTION);
importJson(json, keyspace, cfamily, ssTable);
System.exit(0);
}
}