diff --git a/bin/json2sstable b/bin/json2sstable new file mode 100755 index 0000000000..71965eb69c --- /dev/null +++ b/bin/json2sstable @@ -0,0 +1,42 @@ +#!/bin/sh + +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +if [ "x$CASSANDRA_INCLUDE" = "x" ]; then + for include in /usr/share/cassandra/cassandra.in.sh \ + /usr/local/share/cassandra/cassandra.in.sh \ + /opt/cassandra/cassandra.in.sh \ + ~/.cassandra.in.sh \ + `dirname $0`/cassandra.in.sh; do + if [ -r $include ]; then + . $include + break + fi + done +elif [ -r $CASSANDRA_INCLUDE ]; then + . $CASSANDRA_INCLUDE +fi + +if [ -z $CLASSPATH ]; then + echo "You must set the CLASSPATH var" >&2 + exit 1 +fi + +java -cp $CLASSPATH -Dstorage-config=$CASSANDRA_CONF \ + org.apache.cassandra.tools.SSTableImport "$@" + +# vi:ai sw=4 ts=4 tw=0 et diff --git a/src/java/org/apache/cassandra/tools/SSTableImport.java b/src/java/org/apache/cassandra/tools/SSTableImport.java new file mode 100644 index 0000000000..de049b7119 --- /dev/null +++ b/src/java/org/apache/cassandra/tools/SSTableImport.java @@ -0,0 +1,193 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.cassandra.tools; + +import java.io.FileReader; +import java.io.IOException; +import java.util.*; +import org.apache.cassandra.config.DatabaseDescriptor; +import org.apache.cassandra.db.ColumnFamily; +import org.apache.cassandra.db.DecoratedKey; +import org.apache.cassandra.db.filter.QueryPath; +import org.apache.cassandra.dht.IPartitioner; +import org.apache.cassandra.io.DataOutputBuffer; +import org.apache.cassandra.io.SSTableWriter; +import org.apache.cassandra.utils.FBUtilities; +import org.apache.commons.cli.CommandLine; +import org.apache.commons.cli.CommandLineParser; +import org.apache.commons.cli.Option; +import org.apache.commons.cli.Options; +import org.apache.commons.cli.PosixParser; +import org.json.simple.JSONObject; +import org.json.simple.JSONValue; +import org.json.simple.parser.ParseException; + +/** + * Create SSTables from JSON input + */ +public class SSTableImport +{ + private static final String KEYSPACE_OPTION = "K"; + private static final String COLFAM_OPTION = "c"; + private static Options options; + private static CommandLine cmd; + + static + { + options = new Options(); + Option optKeyspace = new Option(KEYSPACE_OPTION, true, "Keyspace name"); + optKeyspace.setRequired(true); + options.addOption(optKeyspace); + Option optColfamily = new Option(COLFAM_OPTION, true, "Column family"); + optColfamily.setRequired(true); + options.addOption(optColfamily); + } + + /** + * Add columns to a column family. + * + * @param row the columns associated with a row + * @param cfamily the column family to add columns to + */ + private static void addToStandardCF(JSONObject row, ColumnFamily cfamily) + { + for (Map.Entry col : (Set>) row.entrySet()) + { + QueryPath path = new QueryPath(cfamily.name(), null, col.getKey().getBytes()); + byte[] value = FBUtilities.hexToBytes(col.getValue()); + cfamily.addColumn(path, value, System.currentTimeMillis()); + } + } + + /** + * Add super columns to a column family. + * + * @param row the super columns associated with a row + * @param cfamily the column family to add columns to + */ + private static void addToSuperCF(JSONObject row, ColumnFamily cfamily) + { + // Super columns + for (Map.Entry entry : (Set>)row.entrySet()) + { + byte[] superName = entry.getKey().getBytes(); + + // Sub-columns + for (Map.Entry col : (Set>)entry.getValue().entrySet()) + { + QueryPath path = new QueryPath(cfamily.name(), superName, col.getKey().getBytes()); + byte[] value = FBUtilities.hexToBytes(col.getValue()); + cfamily.addColumn(path, value, System.currentTimeMillis()); + } + } + } + + /** + * Convert a JSON formatted file to an SSTable. + * + * @param jsonFile the file containing JSON formatted data + * @param keyspace keyspace the data belongs to + * @param cf column family the data belongs to + * @param ssTablePath file to write the SSTable to + * @throws IOException for errors reading/writing input/output + * @throws ParseException for errors encountered parsing JSON input + */ + private static void importJson(String jsonFile, String keyspace, String cf, String ssTablePath) + throws IOException, ParseException + { + ColumnFamily cfamily = ColumnFamily.create(keyspace, cf); + String cfType = cfamily.type(); // Super or Standard + IPartitioner partitioner = DatabaseDescriptor.getPartitioner(); + DataOutputBuffer dob = new DataOutputBuffer(); + + try + { + JSONObject json = (JSONObject)JSONValue.parse(new FileReader(jsonFile)); + SSTableWriter writer = new SSTableWriter(ssTablePath, json.size(), partitioner); + List> decoratedKeys = new ArrayList>(); + + for (String key : (Set)json.keySet()) + decoratedKeys.add(partitioner.decorateKey(key)); + Collections.sort(decoratedKeys); + + for (DecoratedKey rowKey : decoratedKeys) + { + JSONObject value = (JSONObject)json.get(rowKey.key); + + if (cfType.equals("Super")) + addToSuperCF(value, cfamily); + else + addToStandardCF(value, cfamily); + + ColumnFamily.serializer().serializeWithIndexes(cfamily, dob); + writer.append(rowKey, dob); + dob.reset(); + cfamily.clear(); + } + + writer.closeAndOpenReader(0); + } + catch (ClassCastException cce) + { + //throw cce; + throw new RuntimeException("Invalid JSON input, or incorrect column family"); + } + } + + /** + * Converts JSON to an SSTable file. JSON input can either be a file specified + * using an optional command line argument, or supplied on standard in. + * + * @param args command line arguments + * @throws IOException on failure to open/read/write files or output streams + * @throws ParseException on failure to parse JSON input + */ + public static void main(String[] args) throws IOException, ParseException + { + String usage = String.format("Usage: %s -K keyspace -c column_family %n", + SSTableImport.class.getName()); + + CommandLineParser parser = new PosixParser(); + try + { + cmd = parser.parse(options, args); + } catch (org.apache.commons.cli.ParseException e1) + { + System.err.println(e1.getMessage()); + System.err.println(usage); + System.exit(1); + } + + if (cmd.getArgs().length != 2) + { + System.err.println(usage); + System.exit(1); + } + + String json = cmd.getArgs()[0]; + String ssTable = cmd.getArgs()[1]; + String keyspace = cmd.getOptionValue(KEYSPACE_OPTION); + String cfamily = cmd.getOptionValue(COLFAM_OPTION); + + importJson(json, keyspace, cfamily, ssTable); + + System.exit(0); + } + +} \ No newline at end of file