From e65ce8dcd7c5d212c5325c57d5fae39c6bbca502 Mon Sep 17 00:00:00 2001 From: Eric Evans Date: Mon, 30 Nov 2009 03:25:15 +0000 Subject: [PATCH] JSON importer/exporter tests git-svn-id: https://svn.apache.org/repos/asf/incubator/cassandra/trunk@885325 13f79535-47bb-0310-9956-ffa450edef68 --- build.xml | 3 + .../apache/cassandra/io/SSTableReader.java | 4 +- test/conf/storage-conf.xml | 1 + test/resources/SimpleCF.json | 4 + test/resources/SuperCF.json | 4 + test/unit/org/apache/cassandra/Util.java | 9 ++ .../apache/cassandra/io/SSTableAccessor.java | 35 +++++ .../cassandra/tools/SSTableExportTest.java | 121 ++++++++++++++++++ .../cassandra/tools/SSTableImportTest.java | 67 ++++++++++ 9 files changed, 246 insertions(+), 2 deletions(-) create mode 100644 test/resources/SimpleCF.json create mode 100644 test/resources/SuperCF.json create mode 100644 test/unit/org/apache/cassandra/io/SSTableAccessor.java create mode 100644 test/unit/org/apache/cassandra/tools/SSTableExportTest.java create mode 100644 test/unit/org/apache/cassandra/tools/SSTableImportTest.java diff --git a/build.xml b/build.xml index 80c78e112c..0670ad28dc 100644 --- a/build.xml +++ b/build.xml @@ -214,6 +214,9 @@ + + + diff --git a/src/java/org/apache/cassandra/io/SSTableReader.java b/src/java/org/apache/cassandra/io/SSTableReader.java index e7e2d93423..d9cb340cdc 100644 --- a/src/java/org/apache/cassandra/io/SSTableReader.java +++ b/src/java/org/apache/cassandra/io/SSTableReader.java @@ -186,7 +186,7 @@ public class SSTableReader extends SSTable implements Comparable return indexPositions; } - private void loadBloomFilter() throws IOException + void loadBloomFilter() throws IOException { DataInputStream stream = new DataInputStream(new FileInputStream(filterFilename())); try @@ -199,7 +199,7 @@ public class SSTableReader extends SSTable implements Comparable } } - private void loadIndexFile() throws IOException + void loadIndexFile() throws IOException { BufferedRandomAccessFile input = new BufferedRandomAccessFile(indexFilename(), "r"); try diff --git a/test/conf/storage-conf.xml b/test/conf/storage-conf.xml index dc69a880eb..ed74d568ba 100644 --- a/test/conf/storage-conf.xml +++ b/test/conf/storage-conf.xml @@ -50,6 +50,7 @@ + diff --git a/test/resources/SimpleCF.json b/test/resources/SimpleCF.json new file mode 100644 index 0000000000..2604319854 --- /dev/null +++ b/test/resources/SimpleCF.json @@ -0,0 +1,4 @@ +{ + "rowA": [["colAA", "76616c4141", 1, false], ["colAB", "76616c4142", 1, false]], + "rowB": [["colBA", "76616c4241", 1, false], ["colBB", "76616c4242", 1, false]] +} diff --git a/test/resources/SuperCF.json b/test/resources/SuperCF.json new file mode 100644 index 0000000000..9699120174 --- /dev/null +++ b/test/resources/SuperCF.json @@ -0,0 +1,4 @@ +{ + "rowA": {"superA": {"deletedAt": -9223372036854775808, "subColumns": [["colAA", "76616c75654141", 1, false], ["colAB", "76616c75654142", 1, false]]}}, + "rowB": {"superB": {"deletedAt": -9223372036854775808, "subColumns": [["colBA", "76616c75654241", 1, false], ["colBB", "76616c75654242", 1, false]]}} +} diff --git a/test/unit/org/apache/cassandra/Util.java b/test/unit/org/apache/cassandra/Util.java index 9582ae6f59..cc7717d0a0 100644 --- a/test/unit/org/apache/cassandra/Util.java +++ b/test/unit/org/apache/cassandra/Util.java @@ -21,6 +21,8 @@ package org.apache.cassandra; */ +import java.io.File; +import java.io.IOException; import java.nio.ByteBuffer; import org.apache.cassandra.db.Column; @@ -46,4 +48,11 @@ public class Util bb.putLong(v); return bytes; } + + public static File createTemporarySSTable(String keyspace, String colFam) throws IOException + { + File tmpDir = new File(System.getProperty("java.io.tmpdir") + File.separator + keyspace); + tmpDir.mkdirs(); // Create the per-keyspace temp directory + return File.createTempFile(colFam + "-", "-Data.db", tmpDir); + } } diff --git a/test/unit/org/apache/cassandra/io/SSTableAccessor.java b/test/unit/org/apache/cassandra/io/SSTableAccessor.java new file mode 100644 index 0000000000..e5e4cb8b79 --- /dev/null +++ b/test/unit/org/apache/cassandra/io/SSTableAccessor.java @@ -0,0 +1,35 @@ +/* +* Licensed to the Apache Software Foundation (ASF) under one +* or more contributor license agreements. See the NOTICE file +* distributed with this work for additional information +* regarding copyright ownership. The ASF licenses this file +* to you under the Apache License, Version 2.0 (the +* "License"); you may not use this file except in compliance +* with the License. You may obtain a copy of the License at +* +* http://www.apache.org/licenses/LICENSE-2.0 +* +* Unless required by applicable law or agreed to in writing, +* software distributed under the License is distributed on an +* "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +* KIND, either express or implied. See the License for the +* specific language governing permissions and limitations +* under the License. +*/ +package org.apache.cassandra.io; + +import java.io.IOException; + +import org.apache.cassandra.dht.IPartitioner; + +public class SSTableAccessor +{ + public static SSTableReader getSSTableReader(String filename, IPartitioner partitioner) + throws IOException + { + SSTableReader sstable = new SSTableReader(filename, partitioner, null, null, null); + sstable.loadBloomFilter(); + sstable.loadIndexFile(); + return sstable; + } +} diff --git a/test/unit/org/apache/cassandra/tools/SSTableExportTest.java b/test/unit/org/apache/cassandra/tools/SSTableExportTest.java new file mode 100644 index 0000000000..59913baf94 --- /dev/null +++ b/test/unit/org/apache/cassandra/tools/SSTableExportTest.java @@ -0,0 +1,121 @@ +/* +* Licensed to the Apache Software Foundation (ASF) under one +* or more contributor license agreements. See the NOTICE file +* distributed with this work for additional information +* regarding copyright ownership. The ASF licenses this file +* to you under the Apache License, Version 2.0 (the +* "License"); you may not use this file except in compliance +* with the License. You may obtain a copy of the License at +* +* http://www.apache.org/licenses/LICENSE-2.0 +* +* Unless required by applicable law or agreed to in writing, +* software distributed under the License is distributed on an +* "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +* KIND, either express or implied. See the License for the +* specific language governing permissions and limitations +* under the License. +*/ +package org.apache.cassandra.tools; + +import java.io.File; +import java.io.FileReader; +import java.io.IOException; +import java.io.PrintStream; +import java.util.Arrays; +import org.apache.cassandra.config.DatabaseDescriptor; +import org.apache.cassandra.db.ColumnFamily; +import org.apache.cassandra.db.filter.QueryPath; +import org.apache.cassandra.dht.IPartitioner; +import org.apache.cassandra.io.DataOutputBuffer; +import org.apache.cassandra.io.SSTableReader; +import org.apache.cassandra.io.SSTableWriter; +import static org.apache.cassandra.Util.createTemporarySSTable; +import static org.apache.cassandra.utils.FBUtilities.hexToBytes; +import org.json.simple.JSONArray; +import org.json.simple.JSONObject; +import org.json.simple.JSONValue; +import org.junit.Test; + +public class SSTableExportTest +{ + @Test + public void testExportSimpleCf() throws IOException + { + File tempSS = createTemporarySSTable("Keyspace1", "Standard1"); + ColumnFamily cfamily = ColumnFamily.create("Keyspace1", "Standard1"); + IPartitioner partitioner = DatabaseDescriptor.getPartitioner(); + DataOutputBuffer dob = new DataOutputBuffer(); + SSTableWriter writer = new SSTableWriter(tempSS.getPath(), 2, partitioner); + + // Add rowA + cfamily.addColumn(new QueryPath("Standard1", null, "colA".getBytes()), "valA".getBytes(), 1, false); + ColumnFamily.serializer().serializeWithIndexes(cfamily, dob); + writer.append(partitioner.decorateKey("rowA"), dob); + dob.reset(); + cfamily.clear(); + + // Add rowB + cfamily.addColumn(new QueryPath("Standard1", null, "colB".getBytes()), "valB".getBytes(), 1, false); + ColumnFamily.serializer().serializeWithIndexes(cfamily, dob); + writer.append(partitioner.decorateKey("rowB"), dob); + dob.reset(); + cfamily.clear(); + + SSTableReader reader = writer.closeAndOpenReader(0); + + // Export to JSON and verify + File tempJson = File.createTempFile("Standard1", ".json"); + SSTableExport.export(reader, new PrintStream(tempJson.getPath())); + + JSONObject json = (JSONObject)JSONValue.parse(new FileReader(tempJson)); + + JSONArray rowA = (JSONArray)json.get("rowA"); + JSONArray colA = (JSONArray)rowA.get(0); + assert Arrays.equals(hexToBytes((String)colA.get(1)), "valA".getBytes()); + + JSONArray rowB = (JSONArray)json.get("rowB"); + JSONArray colB = (JSONArray)rowB.get(0); + assert !(Boolean)colB.get(3); + } + + @Test + public void testExportSuperCf() throws IOException + { + File tempSS = createTemporarySSTable("Keyspace1", "Super4"); + ColumnFamily cfamily = ColumnFamily.create("Keyspace1", "Super4"); + IPartitioner partitioner = DatabaseDescriptor.getPartitioner(); + DataOutputBuffer dob = new DataOutputBuffer(); + SSTableWriter writer = new SSTableWriter(tempSS.getPath(), 2, partitioner); + + // Add rowA + cfamily.addColumn(new QueryPath("Super4", "superA".getBytes(), "colA".getBytes()), "valA".getBytes(), 1, false); + ColumnFamily.serializer().serializeWithIndexes(cfamily, dob); + writer.append(partitioner.decorateKey("rowA"), dob); + dob.reset(); + cfamily.clear(); + + // Add rowB + cfamily.addColumn(new QueryPath("Super4", "superB".getBytes(), "colB".getBytes()), "valB".getBytes(), 1, false); + ColumnFamily.serializer().serializeWithIndexes(cfamily, dob); + writer.append(partitioner.decorateKey("rowB"), dob); + dob.reset(); + cfamily.clear(); + + SSTableReader reader = writer.closeAndOpenReader(0); + + // Export to JSON and verify + File tempJson = File.createTempFile("Super4", ".json"); + SSTableExport.export(reader, new PrintStream(tempJson.getPath())); + + JSONObject json = (JSONObject)JSONValue.parse(new FileReader(tempJson)); + + JSONObject rowA = (JSONObject)json.get("rowA"); + JSONObject superA = (JSONObject)rowA.get(cfamily.getComparator().getString("superA".getBytes())); + JSONArray subColumns = (JSONArray)superA.get("subColumns"); + JSONArray colA = (JSONArray)subColumns.get(0); + + assert Arrays.equals(hexToBytes((String)colA.get(1)), "valA".getBytes()); + assert !(Boolean)colA.get(3); + } +} diff --git a/test/unit/org/apache/cassandra/tools/SSTableImportTest.java b/test/unit/org/apache/cassandra/tools/SSTableImportTest.java new file mode 100644 index 0000000000..854f7d4a51 --- /dev/null +++ b/test/unit/org/apache/cassandra/tools/SSTableImportTest.java @@ -0,0 +1,67 @@ +/* +* Licensed to the Apache Software Foundation (ASF) under one +* or more contributor license agreements. See the NOTICE file +* distributed with this work for additional information +* regarding copyright ownership. The ASF licenses this file +* to you under the Apache License, Version 2.0 (the +* "License"); you may not use this file except in compliance +* with the License. You may obtain a copy of the License at +* +* http://www.apache.org/licenses/LICENSE-2.0 +* +* Unless required by applicable law or agreed to in writing, +* software distributed under the License is distributed on an +* "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +* KIND, either express or implied. See the License for the +* specific language governing permissions and limitations +* under the License. +*/ +package org.apache.cassandra.tools; + +import java.io.File; +import java.io.IOException; +import java.util.Arrays; +import org.apache.cassandra.config.DatabaseDescriptor; +import org.apache.cassandra.db.ColumnFamily; +import org.apache.cassandra.db.IColumn; +import org.apache.cassandra.db.filter.NamesQueryFilter; +import org.apache.cassandra.db.filter.QueryPath; +import org.apache.cassandra.io.SSTableAccessor; +import org.apache.cassandra.io.SSTableReader; +import static org.apache.cassandra.utils.FBUtilities.hexToBytes; +import static org.apache.cassandra.Util.createTemporarySSTable; +import org.json.simple.parser.ParseException; +import org.junit.Test; + +public class SSTableImportTest +{ + @Test + public void testImportSimpleCf() throws IOException, ParseException + { + // Import JSON to temp SSTable file + String jsonUrl = getClass().getClassLoader().getResource("resources/SimpleCF.json").getPath(); + File tempSS = createTemporarySSTable("Keyspace1", "Standard1"); + SSTableImport.importJson(jsonUrl, "Keyspace1", "Standard1", tempSS.getPath()); + + // Verify results + SSTableReader reader = SSTableAccessor.getSSTableReader(tempSS.getPath(), DatabaseDescriptor.getPartitioner()); + NamesQueryFilter qf = new NamesQueryFilter("rowA", new QueryPath("Standard1", null, null), "colAA".getBytes()); + ColumnFamily cf = qf.getSSTableColumnIterator(reader).getColumnFamily(); + assert Arrays.equals(cf.getColumn("colAA".getBytes()).value(), hexToBytes("76616c4141")); + } + + @Test + public void testImportSuperCf() throws IOException, ParseException + { + String jsonUrl = getClass().getClassLoader().getResource("resources/SuperCF.json").getPath(); + File tempSS = createTemporarySSTable("Keyspace1", "Super4"); + SSTableImport.importJson(jsonUrl, "Keyspace1", "Super4", tempSS.getPath()); + + // Verify results + SSTableReader reader = SSTableAccessor.getSSTableReader(tempSS.getPath(), DatabaseDescriptor.getPartitioner()); + NamesQueryFilter qf = new NamesQueryFilter("rowA", new QueryPath("Super4", null, null), "superA".getBytes()); + ColumnFamily cf = qf.getSSTableColumnIterator(reader).getColumnFamily(); + IColumn superCol = cf.getColumn("superA".getBytes()); + assert Arrays.equals(superCol.getSubColumn("colAA".getBytes()).value(), hexToBytes("76616c75654141")); + } +}