From 56cfc6ea35d1410f2f5a8ae711ae33342f286d79 Mon Sep 17 00:00:00 2001 From: Aleksey Yeschenko Date: Fri, 16 Oct 2015 13:23:20 +0100 Subject: [PATCH] Remove Pig support patch by Aleksey Yeschenko; reviewed by Sylvain Lebresne for CASSANDRA-10542 --- .rat-excludes | 1 - CHANGES.txt | 1 + NEWS.txt | 2 +- build.xml | 67 +- examples/pig/README.txt | 118 -- examples/pig/bin/pig_cassandra | 46 - examples/pig/example-populate-cql.txt | 24 - examples/pig/example-script-cql.pig | 11 - examples/pig/example-script.pig | 9 - examples/pig/libdata.csv | 200 --- examples/pig/test/populate-cli.txt | 134 -- examples/pig/test/populate-cql.txt | 33 - examples/pig/test/test_cql_storage.pig | 14 - examples/pig/test/test_storage.pig | 85 -- .../hadoop/pig/CqlNativeStorage.java | 1082 ----------------- .../cassandra/hadoop/pig/StorageHelper.java | 122 -- test/conf/cassandra_pig.yaml | 42 - .../cassandra/pig/CqlRecordReaderTest.java | 150 --- .../cassandra/pig/CqlTableDataTypeTest.java | 479 -------- .../apache/cassandra/pig/CqlTableTest.java | 303 ----- .../org/apache/cassandra/pig/PigTestBase.java | 117 -- .../pig/ThriftColumnFamilyDataTypeTest.java | 150 --- .../cassandra/pig/ThriftColumnFamilyTest.java | 617 ---------- test/pig/org/apache/pig/test/MiniCluster.java | 86 -- .../apache/pig/test/MiniGenericCluster.java | 122 -- .../pig/test/WindowsLocalFileSystem.java | 62 - 26 files changed, 11 insertions(+), 4066 deletions(-) delete mode 100644 examples/pig/README.txt delete mode 100755 examples/pig/bin/pig_cassandra delete mode 100644 examples/pig/example-populate-cql.txt delete mode 100644 examples/pig/example-script-cql.pig delete mode 100644 examples/pig/example-script.pig delete mode 100644 examples/pig/libdata.csv delete mode 100644 examples/pig/test/populate-cli.txt delete mode 100644 examples/pig/test/populate-cql.txt delete mode 100644 examples/pig/test/test_cql_storage.pig delete mode 100644 examples/pig/test/test_storage.pig delete mode 100644 src/java/org/apache/cassandra/hadoop/pig/CqlNativeStorage.java delete mode 100644 src/java/org/apache/cassandra/hadoop/pig/StorageHelper.java delete mode 100644 test/conf/cassandra_pig.yaml delete mode 100644 test/pig/org/apache/cassandra/pig/CqlRecordReaderTest.java delete mode 100644 test/pig/org/apache/cassandra/pig/CqlTableDataTypeTest.java delete mode 100644 test/pig/org/apache/cassandra/pig/CqlTableTest.java delete mode 100644 test/pig/org/apache/cassandra/pig/PigTestBase.java delete mode 100644 test/pig/org/apache/cassandra/pig/ThriftColumnFamilyDataTypeTest.java delete mode 100644 test/pig/org/apache/cassandra/pig/ThriftColumnFamilyTest.java delete mode 100644 test/pig/org/apache/pig/test/MiniCluster.java delete mode 100644 test/pig/org/apache/pig/test/MiniGenericCluster.java delete mode 100644 test/pig/org/apache/pig/test/WindowsLocalFileSystem.java diff --git a/.rat-excludes b/.rat-excludes index d95b499154..6df36a5e6a 100644 --- a/.rat-excludes +++ b/.rat-excludes @@ -24,7 +24,6 @@ test/data/serialization/*/* doc/** build.properties.default test/data/** -examples/pig/** examples/triggers/build.xml examples/triggers/conf/* examples/hadoop_word_count/conf/log4j.properties diff --git a/CHANGES.txt b/CHANGES.txt index dcacc6950f..6bdaa04444 100644 --- a/CHANGES.txt +++ b/CHANGES.txt @@ -1,4 +1,5 @@ 3.0-rc2 + * Remove Pig support (CASSANDRA-10542) * Fix LogFile throws Exception when assertion is disabled (CASSANDRA-10522) * Revert CASSANDRA-7486, make CMS default GC, move GC config to conf/jvm.options (CASSANDRA-10403) diff --git a/NEWS.txt b/NEWS.txt index a18ce6b474..48a0733468 100644 --- a/NEWS.txt +++ b/NEWS.txt @@ -84,7 +84,7 @@ Upgrading (schema versions outputted by `nodetool describecluster` are all the same). - Schema metadata is now stored in the new `system_schema` keyspace, and legacy `system.schema_*` tables are now gone; see CASSANDRA-6717 for details. - - Pig's CassandraStorage has been removed. Use CqlNativeStorage instead. + - Pig's support has been removed. - Hadoop BulkOutputFormat and BulkRecordWriter have been removed; use CqlBulkOutputFormat and CqlBulkRecordWriter instead. - Hadoop ColumnFamilyInputFormat and ColumnFamilyOutputFormat have been removed; diff --git a/build.xml b/build.xml index 87f98ccbfa..e039bd2595 100644 --- a/build.xml +++ b/build.xml @@ -63,7 +63,6 @@ - @@ -386,17 +385,12 @@ - - - - - - - + + @@ -467,7 +461,6 @@ - @@ -524,7 +508,6 @@ - @@ -572,7 +555,6 @@ - @@ -694,26 +676,6 @@ - - - - - - - - - - - - - - - - @@ -1352,7 +1311,7 @@ ant testsome -Dtest.name=org.apache.cassandra.service.StorageServiceServerTest -Dtest.methods=testRegularMode,testGetAllRangesEmpty --> - + @@ -1370,7 +1329,7 @@ - + @@ -1489,14 +1448,8 @@ - - - - - @@ -1716,7 +1669,7 @@ - + @@ -1756,7 +1709,6 @@ - @@ -1964,7 +1916,6 @@ - diff --git a/examples/pig/README.txt b/examples/pig/README.txt deleted file mode 100644 index 1553a9f263..0000000000 --- a/examples/pig/README.txt +++ /dev/null @@ -1,118 +0,0 @@ -A Pig storage class that reads all columns from a given ColumnFamily, or writes -properly formatted results into a ColumnFamily. - -Getting Started -=============== - -First build and start a Cassandra server with the default -configuration and set the PIG_HOME and JAVA_HOME environment -variables to the location of a Pig >= 0.7.0 install and your Java -install. - -If you would like to run using the Hadoop backend, you should -also set PIG_CONF_DIR to the location of your Hadoop config. - -Finally, set the following as environment variables (uppercase, -underscored), or as Hadoop configuration variables (lowercase, dotted): -* PIG_INITIAL_ADDRESS or cassandra.thrift.address : initial address to connect to -* PIG_RPC_PORT or cassandra.thrift.port : the port thrift is listening on -* PIG_PARTITIONER or cassandra.partitioner.class : cluster partitioner - -For example, against a local node with the default settings, you'd use: -export PIG_INITIAL_ADDRESS=localhost -export PIG_RPC_PORT=9160 -export PIG_PARTITIONER=org.apache.cassandra.dht.Murmur3Partitioner - -These properties can be overridden with the following if you use different clusters -for input and output: -* PIG_INPUT_INITIAL_ADDRESS : initial address to connect to for reading -* PIG_INPUT_RPC_PORT : the port thrift is listening on for reading -* PIG_INPUT_PARTITIONER : cluster partitioner for reading -* PIG_OUTPUT_INITIAL_ADDRESS : initial address to connect to for writing -* PIG_OUTPUT_RPC_PORT : the port thrift is listening on for writing -* PIG_OUTPUT_PARTITIONER : cluster partitioner for writing - -CassandraStorage -================ - -The CassandraStorage class is for any non-CQL3 ColumnFamilies you may have. For CQL3 support, refer to the CqlNativeStorage section. - -examples/pig$ bin/pig_cassandra -x local example-script.pig - -This will run the test script against your Cassandra instance -and will assume that there is a MyKeyspace/MyColumnFamily with some -data in it. It will run in local mode (see pig docs for more info). - -If you'd like to get to a 'grunt>' shell prompt, run: - -examples/pig$ bin/pig_cassandra -x local - -Once the 'grunt>' shell has loaded, try a simple program like the -following, which will determine the top 50 column names: - -grunt> rows = LOAD 'cassandra://MyKeyspace/MyColumnFamily' USING CassandraStorage(); -grunt> cols = FOREACH rows GENERATE flatten(columns); -grunt> colnames = FOREACH cols GENERATE $0; -grunt> namegroups = GROUP colnames BY (chararray) $0; -grunt> namecounts = FOREACH namegroups GENERATE COUNT($1), group; -grunt> orderednames = ORDER namecounts BY $0 DESC; -grunt> topnames = LIMIT orderednames 50; -grunt> dump topnames; - -Slices on columns can also be specified: -grunt> rows = LOAD 'cassandra://MyKeyspace/MyColumnFamily?slice_start=C2&slice_end=C4&limit=1&reversed=true' USING CassandraStorage(); - -Binary values for slice_start and slice_end can be escaped such as '\u0255' - -Outputting to Cassandra requires the same format from input, so the simplest example is: - -grunt> rows = LOAD 'cassandra://MyKeyspace/MyColumnFamily' USING CassandraStorage(); -grunt> STORE rows into 'cassandra://MyKeyspace/MyColumnFamily' USING CassandraStorage(); - -Which will copy the ColumnFamily. Note that the destination ColumnFamily must -already exist for this to work. - -See the example in test/ to see how schema is inferred. - -Advanced Options for CassandraStorage -===================================== - -The following environment variables default to false but can be set to true to enable them: - -PIG_WIDEROW_INPUT: this enables loading of rows with many columns without - incurring memory pressure. All columns will be in a bag and indexes are not - supported. This can also be set in the LOAD url by adding - the 'widerows=true' parameter. - -PIG_USE_SECONDARY: this allows easy use of secondary indexes within your - script, by appending every index to the schema as 'index_$name', allowing - filtering of loaded rows with a statement like "FILTER rows BY index_color eq - 'blue'" if you have an index called 'color' defined. This - can also be set in the LOAD url by adding the - 'use_secondary=true' parameter. - -PIG_INPUT_SPLIT_SIZE: this sets the split size passed to Hadoop, controlling - the amount of mapper tasks created. This can also be set in the LOAD url by - adding the 'split_size=X' parameter, where X is an integer amount for the size. - -CqlNativeStorage -================ - -The CqlNativeStorage class is somewhat similar to CassandraStorage, but it can work with CQL3-defined ColumnFamilies. The main difference is in the URL format: - -cql://[username:password@]/ - [?[page_size=][&columns=][&output_query=] - [&where_clause=][&split_size=][&partitioner=][&use_secondary=true|false] - [&init_address=][&native_port=][&core_conns=] - [&max_conns=][&min_simult_reqs=][&max_simult_reqs=] - [&native_timeout=][&native_read_timeout=][&rec_buff_size=] - [&send_buff_size=][&solinger=][&tcp_nodelay=][&reuse_address=] - [&keep_alive=][&auth_provider=][&trust_store_path=] - [&key_store_path=][&trust_store_password=] - [&key_store_password=][&cipher_suites=][&input_cql=] - [columns=][where_clause=]] -Which in grunt, the simplest example would look like: - -grunt> rows = LOAD 'cql://MyKeyspace/MyColumnFamily' USING CqlNativeStorage(); - -CqlNativeStorage handles wide rows automatically and thus has no separate flag for this. diff --git a/examples/pig/bin/pig_cassandra b/examples/pig/bin/pig_cassandra deleted file mode 100755 index 5d9888871d..0000000000 --- a/examples/pig/bin/pig_cassandra +++ /dev/null @@ -1,46 +0,0 @@ -#!/bin/sh - -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -cwd=`dirname $0` -cassandra_home="$cwd/../../../" - -# general jars. -for jar in $cassandra_home/lib/*.jar $cassandra_home/build/lib/jars/*.jar $cassandra_home/build/apache-cassandra*.jar; do - CLASSPATH=$CLASSPATH:$jar -done - -if [ "x$PIG_HOME" = "x" ]; then - echo "PIG_HOME not set: requires Pig >= 0.7.0" >&2 - exit 1 -fi - -# pig jar. -for jar in $PIG_HOME/*.jar; do - PIG_JAR=$jar -done -echo "Using $PIG_JAR." -if [ ! -e $PIG_JAR ]; then - echo "Unable to locate Pig jar" >&2 - exit 1 -fi - -CLASSPATH=$CLASSPATH:$PIG_JAR - -export PIG_CLASSPATH=$PIG_CLASSPATH:$CLASSPATH -export PIG_OPTS="$PIG_OPTS -Dudf.import.list=org.apache.cassandra.hadoop.pig" -$PIG_HOME/bin/pig $* diff --git a/examples/pig/example-populate-cql.txt b/examples/pig/example-populate-cql.txt deleted file mode 100644 index 8e23582846..0000000000 --- a/examples/pig/example-populate-cql.txt +++ /dev/null @@ -1,24 +0,0 @@ -CREATE KEYSPACE libdata WITH replication = {'class': 'SimpleStrategy', 'replication_factor': 1 }; -USE libdata; - -CREATE TABLE libout ("STABR" TEXT, "FSCSKEY" TEXT, "FSCS_SEQ" TEXT, - "LIBID" TEXT, "LIBNAME" TEXT, "ADDRESS" TEXT, "CITY" TEXT, - "ZIP" TEXT, "ZIP4" TEXT, "CNTY" TEXT, "PHONE" TEXT, "C_OUT_TY" TEXT, - "C_MSA" TEXT, "SQ_FEET" INT, "F_SQ_FT" TEXT, "L_NUM_BM" INT, - "F_BKMOB" TEXT, "HOURS" INT, "F_HOURS" TEXT, "WKS_OPEN" INT, - "F_WKSOPN" TEXT, "YR_SUB" INT, "STATSTRU" INT, "STATNAME" INT, - "STATADDR" INT, "LONGITUD" FLOAT, "LATITUDE" FLOAT, "FIPSST" INT, - "FIPSCO" INT, "FIPSPLAC" INT, "CNTYPOP" INT, "LOCALE" TEXT, - "CENTRACT" FLOAT, "CENBLOCK" INT, "CDCODE" TEXT, "MAT_CENT" TEXT, - "MAT_TYPE" INT, "CBSA" INT, "MICROF" TEXT, - PRIMARY KEY ("FSCSKEY", "FSCS_SEQ")); - -COPY libout ("STABR","FSCSKEY","FSCS_SEQ","LIBID","LIBNAME", - "ADDRESS","CITY","ZIP","ZIP4","CNTY","PHONE","C_OUT_TY", - "C_MSA","SQ_FEET","F_SQ_FT","L_NUM_BM","F_BKMOB","HOURS", - "F_HOURS","WKS_OPEN","F_WKSOPN","YR_SUB","STATSTRU","STATNAME", - "STATADDR","LONGITUD","LATITUDE","FIPSST","FIPSCO","FIPSPLAC", - "CNTYPOP","LOCALE","CENTRACT","CENBLOCK","CDCODE","MAT_CENT", - "MAT_TYPE","CBSA","MICROF") FROM 'libdata.csv' WITH HEADER=TRUE; - -CREATE TABLE libsqft (year INT, state TEXT, sqft BIGINT, PRIMARY KEY (year, state)); diff --git a/examples/pig/example-script-cql.pig b/examples/pig/example-script-cql.pig deleted file mode 100644 index ef11130955..0000000000 --- a/examples/pig/example-script-cql.pig +++ /dev/null @@ -1,11 +0,0 @@ --- CqlNativeStorage -libdata = LOAD 'cql://libdata/libout' USING CqlNativeStorage(); -book_by_mail = FILTER libdata BY C_OUT_TY == 'BM'; - -libdata_buildings = FILTER libdata BY SQ_FEET > 0; -state_flat = FOREACH libdata_buildings GENERATE STABR AS State,SQ_FEET AS SquareFeet; -state_grouped = GROUP state_flat BY State; -state_footage = FOREACH state_grouped GENERATE group AS State, SUM(state_flat.SquareFeet) AS TotalFeet:int; - -insert_format= FOREACH state_footage GENERATE TOTUPLE(TOTUPLE('year',2011),TOTUPLE('state',State)),TOTUPLE(TotalFeet); -STORE insert_format INTO 'cql://libdata/libsqft?output_query=UPDATE%20libdata.libsqft%20SET%20sqft%20%3D%20%3F' USING CqlNativeStorage; \ No newline at end of file diff --git a/examples/pig/example-script.pig b/examples/pig/example-script.pig deleted file mode 100644 index d47fc28849..0000000000 --- a/examples/pig/example-script.pig +++ /dev/null @@ -1,9 +0,0 @@ --- CassandraStorage -rows = LOAD 'cassandra://MyKeyspace/MyColumnFamily' USING CassandraStorage(); -cols = FOREACH rows GENERATE flatten(columns); -colnames = FOREACH cols GENERATE $0; -namegroups = GROUP colnames BY (chararray) $0; -namecounts = FOREACH namegroups GENERATE COUNT($1), group; -orderednames = ORDER namecounts BY $0 DESC; -topnames = LIMIT orderednames 50; -dump topnames; \ No newline at end of file diff --git a/examples/pig/libdata.csv b/examples/pig/libdata.csv deleted file mode 100644 index f9bd0b6ffd..0000000000 --- a/examples/pig/libdata.csv +++ /dev/null @@ -1,200 +0,0 @@ -KS,KS0189,002,KS0037,CALDWELL PUBLIC LIBRARY,120 S. MAIN ST.,CALDWELL,67022,1414,SUMNER,6208456879,CE,NO,4500,R_11,0,R_11,1200,R_11,50,R_11,2012,00,00,00,-97.606964,37.03116,20,191,09900,23835,43,9623,3078,2004,0,1,48620,0 -CA,CA0152,004,M748,SOLIZ LIBRARY,2820 JOURDAN ST.,OXNARD,93036,1611,VENTURA,8054854515,BR,CC,3030,R_11,0,R_11,1227,R_11,52,R_11,2012,00,00,00,-119.168424,34.2372,06,111,22370,831126,21,50.02999878,2005,0626,0,1,37100,0 -WV,WV0023,002,22,MORGANTOWN PUBLIC LIBRARY,373 SPRUCE STREET,MORGANTOWN,26505,5564,MONONGALIA,3042917425,CE,NO,24140,R_11,0,R_11,3016,R_11,52,R_11,2012,00,00,00,-79.954276,39.630204,54,061,55756,98613,13,107,1011,5401,0,1,34060,0 -IL,IL0603,002,3067300,VESPASIAN WARNER PUBLIC LIBRARY DISTRICT,310 NORTH QUINCY STREET,CLINTON,61727,1300,DE WITT,2179355174,CE,NO,36000,R_11,0,R_11,3172,R_11,-1,U_11,2012,00,00,00,-88.961745,40.154608,17,039,15001,16528,32,9717,2032,1713,0,1,14010,0 -NC,NC0032,017,C-FORSYTH-Y,YWCA BEST CHOICE CENTER,1031 HIGHLAND AVE,WINSTON-SALEM,27101,3108,FORSYTH,3367220597,BR,CC,750,R_11,0,R_11,312,R_11,52,R_11,2012,00,00,00,-80.233147,36.108588,37,067,75000,354454,12,6,2018,3712,0,1,49180,0 -NY,NY0578,002,7200446540,TUXEDO PARK LIBRARY,227 ROUTE 17,TUXEDO PARK,10987,4405,ORANGE,8453512207,CE,NC,9000,R_11,0,R_11,3094,R_11,52,R_11,2012,00,00,00,-74.18461,41.193192,36,071,00000,374438,41,149,3005,3618,0,5,35620,0 -IL,IL0508,002,3063300,STEELEVILLE AREA PUBLIC LIBRARY DISTRICT,625 SOUTH SPARTA STREET,STEELEVILLE,62288,2147,RANDOLPH,6189659732,CE,NO,4998,R_11,0,R_11,2630,R_11,-1,U_11,2012,00,00,00,-89.658669,38.003021,17,157,72468,33262,32,9511,4060,1712,0,1,0,. -NY,NY0601,002,7200592060,SUNSHINE HALL FREE LIBRARY,14 PROCTOR ROAD,ELDRED,12732,0157,SULLIVAN,8455576258,CE,NO,2300,R_11,0,R_11,1250,R_11,52,R_11,2012,00,00,00,-74.882528,41.526061,36,105,00000,77079,42,9524,1021,3619,0,1,0,. -FL,FL0001,012,FL0001-012,REDDICK PUBLIC LIBRARY,15150 N.W. GAINESVILLE ROAD,REDDICK,32686,3221,MARION,3524382566,BR,NC,2496,R_11,0,R_11,1228,R_11,52,R_11,2012,00,00,00,-82.197543,29.369123,12,083,59675,332472,42,2,2056,1205,0,1,36100,0 -NC,NC0010,009,R-HYCONEECHEE-CC,CARRBORO CYBRARY,100 N GREENSBORO ST,CARRBORO,27510,2016,ORANGE,9199187387,BR,NC,1060,R_11,0,R_11,1560,R_11,52,R_11,2012,00,00,15,-79.072256,35.910375,37,135,10620,135418,21,107.0500031,3021,3704,0,1,20500,0 -WV,WV0077,004,076B,COAL RIVER,494 JOHN SLACK CIRCLE,RACINE,25165,M,BOONE,3048378437,BR,NO,1200,R_11,0,R_11,1820,R_11,52,R_11,2012,00,00,00,-81.656444,38.145905,54,005,00000,24459,41,9582,3074,5403,X,14,16620,0 -TX,TX0279,002,360,SHEPHERD PUBLIC LIBRARY,30 N LIBERTY ST,SHEPHERD,77371,2460,SAN JACINTO,9366283515,CE,NC,2668,R_11,0,R_11,1626,R_11,52,R_11,2012,00,00,00,-94.997618,30.498398,48,407,67424,26876,42,2001.01001,1011,4808,0,1,0,. -MI,MI0261,002,MI0261-002,SHIAWASSEE DISTRICT LIBRARY,502 WEST MAIN STREET,OWOSSO,48867,2687,SHIAWASSEE,9897255134,CE,NO,8400,R_11,0,R_11,2868,R_11,52,R_11,2012,00,00,00,-84.177198,42.997558,26,155,61940,69934,32,306,1007,2604,0,1,37020,1 -NE,NE0210,002,235T,SPENCER TOWNSHIP LIBRARY,110 WEST MAIN STREET,SPENCER,68777,0189,BOYD,4025891331,CE,NO,1750,R_11,0,R_11,546,R_11,52,R_11,2012,00,00,00,-98.701432,42.874026,31,015,46275,2081,43,9758,1234,3103,0,1,0,. -ND,ND0012,002,ND0012-002,CARNEGIE REGIONAL LIBRARY,49 WEST 7TH STREET,GRAFTON,58237,1050,WALSH,7013522754,CE,CC,8692,R_11,0,R_11,2000,R_11,50,R_11,2012,00,00,00,-97.41183,48.417585,38,099,31820,11032,33,9580,1077,3800,0,1,0,. -CT,CT0193,002,3900,EASTFORD PUBLIC LIBRARY,179 EASTFORD RD.,EASTFORD,06242,0908,WINDHAM,8609740125,CE,NO,1512,R_11,0,R_11,1216,R_11,52,R_11,2012,00,00,00,-72.080012,41.901001,09,015,00000,118205,42,9022,1049,0902,0,1,49340,0 -IL,IL0421,002,3053900,PECATONICA PUBLIC LIBRARY DISTRICT,400 WEST ELEVENTH STREET,PECATONICA,61063,9173,WINNEBAGO,8152392616,CE,NC,11800,R_11,0,R_11,2080,R_11,-1,U_11,2012,00,00,00,-89.365175,42.304798,17,201,58408,293660,42,43,2010,1716,0,1,40420,0 -AR,AR0019,002,AR019-002,LYDA MILLER PUBLIC LIBRARY,2609 HIGHWAY 367 S,BALD KNOB,72010,0287,WHITE,5017245452,BR,NO,2100,R_11,0,R_11,-1,U_11,52,R_11,2012,00,06,00,-91.563371,35.315974,05,145,03280,78089,32,706,4050,0502,0,1,42620,1 -TX,TX0473,002,513,ELGIN PUBLIC LIBRARY,404 N MAIN ST,ELGIN,78621,2625,BASTROP,5122815678,CE,NC,12738,R_11,0,R_11,1983,R_11,52,R_11,2012,00,00,00,-97.371621,30.350555,48,021,23044,75028,32,9502,2001,4810,0,1,12420,0 -MA,MA0164,002,MALDEN,MALDEN PUBLIC LIBRARY,36 SALEM ST.,MALDEN,02148,5208,MIDDLESEX,7813240218,CE,NC,44000,R_11,0,R_11,3126,R_11,52,R_11,2012,00,00,00,-71.066333,42.427158,25,017,37875,1522050,21,3416,5007,2505,0,1,14460,0 -WI,WI0409,002,WI0409,LOWELL PUBLIC LIBRARY,105 N. RIVER ST.,LOWELL,53557,0015,DODGE,9209275700,CE,NO,900,R_11,0,R_11,1040,R_11,52,R_11,2012,00,00,00,-88.79318,43.345758,55,027,00000,88721,42,9618,1101,5505,X,15,13180,1 -PA,PA0413,002,901630213,BURGETTSTOWN COMMUNITY LIBRARY,2 KERR ST,BURGETTSTOWN,15021,1127,WASHINGTON,7249479780,CE,NC,6781,R_11,0,R_11,2392,R_11,52,R_11,2012,00,00,00,-80.389683,40.379256,42,125,10224,208170,42,7127,1030,4218,0,1,38300,0 -NE,NE0091,002,101C,GILBERT PUBLIC LIBRARY,628 SECOND STREET,FRIEND,68359,1308,SALINE,4029475081,CE,NO,1678,R_11,0,R_11,1196,R_11,52,R_11,2012,00,00,00,-97.285879,40.653724,31,151,17775,14398,43,9607,2058,3103,0,1,0,. -KS,KS0028,002,KS0160,KANOPOLIS PUBLIC LIBRARY,221 N. KANSAS,KANOPOLIS,67454,0205,ELLSWORTH,7854723053,CE,NO,897,R_11,0,R_11,832,R_11,52,R_11,2012,00,00,00,-98.157901,38.711743,20,053,35950,6478,42,866,2294,2001,0,1,0,. -IL,IL0098,059,3010857,PORTAGE-CRAGIN BRANCH,5108 WEST BELMONT AVENUE,CHICAGO,60641,4206,COOK,3127440152,BR,CC,14000,R_11,0,R_11,2496,R_11,52,R_11,2012,00,00,00,-87.754563,41.938765,17,031,14000,5214098,11,1510.01001,2008,1704,0,1,16980,0 -PA,PA0083,002,905620545,SHEFFIELD TOWNSHIP LIBRARY,20 LEATHER ST,SHEFFIELD,16347,0607,WARREN,8149683439,CE,NO,1131,R_11,0,R_11,1352,R_11,52,R_11,2012,00,00,00,-79.033263,41.701956,42,123,69984,41480,42,9712,3019,4205,0,1,47620,1 -UT,UT0072,001,UT0072-001,SALEM CITY LIBRARY,59 SOUTH MAIN STREET,SALEM,84653,9601,UTAH,8014232622,CE,NC,4150,R_11,0,R_11,2216,R_11,52,R_11,2012,00,00,00,-111.673401,40.053702,49,049,65770,530104,32,104.0699997,2016,4903,0,1,39340,0 -NJ,NJ0048,002,NJ0048-002,ORADELL PUBLIC LIBRARY,375 KINDERKAMACK ROAD,ORADELL,07649,2122,BERGEN,2012622613,CE,NC,16000,R_11,0,R_11,3016,R_11,52,R_11,2012,00,00,00,-74.031827,40.953183,34,003,54990,912773,21,400.019989,4006,3405,0,1,35620,0 -WI,WI0191,002,WI0191,MENOMONEE FALLS PUBLIC LIBRARY,W156N8436 PILGRIM RD.,MENOMONEE FALLS,53051,3140,WAUKESHA,2625328930,CE,NC,52951,R_11,0,R_11,3328,R_11,52,R_11,2012,00,00,00,-88.103683,43.170852,55,133,51000,390701,21,2001.030029,3023,5505,0,1,33340,0 -NC,NC0033,002,C-FRANKLIN-F,FRANKLIN COUNTY LIBRARY,906 N MAIN ST,LOUISBURG,27549,2199,FRANKLIN,9194962111,CE,NC,6000,R_11,0,R_11,2704,R_11,52,R_11,2012,00,00,15,-78.296077,36.112774,37,069,39360,61181,32,603.0100098,1018,3701,0,1,39580,0 -FL,FL0042,006,FL0042-006,DUNBAR JUPITER HAMMON PUBLIC LIBRARY,3095 BLOUNT STREET,FORT MYERS,33916,4100,LEE,2395334150,BR,NC,11400,R_11,0,R_11,1992,R_11,52,R_11,2012,00,00,00,-81.84986,26.642837,12,071,24125,631602,13,5.019999981,2011,1219,0,1,15980,0 -LA,LA0038,005,LA0038,VERNON PARISH LIBRARY BOOKMOBILE,1401 NOLAN TRACE,LEESVILLE,71446,8378,VERNON,3372392027,BS,NO,-3,U_11,1,R_11,1248,R_11,52,R_11,2012,00,00,00,-93.133117,31.328122,22,115,00000,52178,43,9501,1052,2204,2,15,22860,1 -FL,FL0058,002,FL0058-002,NORTH PALM BEACH PUBLIC LIBRARY,303 ANCHORAGE DRIVE,NORTH PALM BEACH,33408,4990,PALM BEACH,5618413383,CE,NC,24893,R_11,0,R_11,3024,R_11,52,R_11,2012,00,00,00,-80.057184,26.810361,12,099,49600,1337512,21,7.019999981,1015,1218,0,1,33100,0 -MO,MO0037,002,MO0036-002,UNIVERSITY CITY PUBLIC LIBRARY,6701 DELMAR BLVD,UNIVERSITY CITY,63130,3199,ST. LOUIS,3147273150,CE,NC,43000,R_11,0,R_11,3744,R_11,52,R_11,2012,00,00,00,-90.308719,38.656351,29,189,75220,999321,21,2161,2010,2901,0,1,41180,0 -TX,TX0582,002,575,MCMULLEN MEMORIAL LIBRARY,900 N MAIN ST,HUNTINGTON,75949,M,ANGELINA,9368764516,CE,NO,2500,R_11,0,R_11,1792,R_11,52,R_11,2012,00,00,00,-94.572496,31.285469,48,005,35492,87276,42,13,1016,4801,0,1,31260,1 -CT,CT0109,002,9902,ATWATER MEMORIAL LIBRARY,1720 FOXON RD.,NORTH BRANFORD,06471,0258,NEW HAVEN,2033156020,CE,NC,13000,R_11,0,R_11,2830,R_11,52,R_11,2012,00,00,00,-72.765106,41.328739,09,009,00000,862812,21,1861,1010,0903,0,1,35300,0 -AZ,AZ0064,011,BR-GEASA,GEASA-MARANA BRANCH LIBRARY,13370 LON ADAMS ROAD,MARANA,85653,9050,PIMA,5205945255,BR,NC,2900,R_11,0,R_11,2034,R_11,52,R_11,2012,00,00,00,-111.208324,32.449152,04,019,44270,987573,42,44.29999924,2025,0401,0,1,46060,0 -TX,TX0216,005,30.53,THEODORE JOHNS BRANCH LIBRARY,4255 FANNETT RD,BEAUMONT,77705,M,JEFFERSON,4098425223,BR,CC,11970,R_11,0,R_11,3132,R_11,52,R_11,2012,00,00,00,-94.129488,30.045033,48,245,07000,253160,12,22,2023,4814,0,1,13140,0 -LA,LA0044,012,49,GENEALOGY BRANCH & ADM. OFFICE,200 EAST MULBERRY STREET,AMITE,70422,2524,TANGIPAHOA,9857487559,BR,NO,8000,R_11,0,R_11,1716,R_11,52,R_11,2012,00,00,00,-90.506645,30.728051,22,105,01885,122519,32,9534,3091,2205,0,1,25220,0 -MD,MD0024,005,MD0024-005,POCOMOKE BRANCH,301 MARKET ST.,POCOMOKE CITY,21851,1113,WORCESTER,4109570878,BR,NO,6728,R_11,0,R_11,2136,R_11,52,R_11,2012,00,00,00,-75.566713,38.074163,24,047,62475,51446,32,9515,3032,2401,0,1,41540,0 -IL,IL0318,003,3043000,MARION CARNEGIE LIBRARY,206 SOUTH MARKET STREET,MARION,62959,2516,WILLIAMSON,6189935935,CE,NO,21000,R_11,0,R_11,3129,R_11,-1,U_11,2012,00,00,00,-88.927162,37.729836,17,199,46916,66680,33,211,3029,1712,0,1,16060,0 -IA,IA0307,002,RD,FONTANELLE PUBLIC LIBRARY,303 WASHINGTON,FONTANELLE,50846,0387,ADAIR,6417454981,CE,NO,2000,R_11,0,R_11,1040,R_11,52,R_11,2012,00,00,00,-94.563977,41.289591,19,001,28290,7572,43,9603,4125,1903,0,1,0,. -PR,PR0010,001,BP0009,MOROVIS ELECTRONIC MUNICIPAL LIBRARY,"JOSE DEL RIO ST., CORNER OF BETANCE",MOROVIS,00687,M,MOROVIS,7878626161,CE,NO,-1,U_11,0,R_11,-1,U_11,-1,U_11,2012,00,00,00,-66.427046,18.340777,72,101,29555,32753,21,9553,3018,7298,X,14,41980,0 -OH,OH0188,002,288C,IDA RUPP PUBLIC LIBRARY,310 MADISON ST.,PORT CLINTON,43452,1921,OTTAWA,4197323212,CE,NC,17075,R_11,0,R_11,3119,R_11,52,R_11,2012,00,00,00,-82.941131,41.510222,39,123,64150,41435,31,506,2000,3909,0,1,38840,1 -MD,MD0008,004,MD0008-004,NORTH EAST BRANCH,106 W. CECIL AVE.,NORTH EAST,21901,3506,CECIL,4109966269,BR,NO,2800,R_11,0,R_11,2600,R_11,52,R_11,2012,00,00,00,-75.945126,39.60096,24,015,56450,101628,31,309.0299988,1067,2401,0,1,37980,0 -LA,LA0055,007,59,DELMONT GARDENS BRANCH,3351 LORRAINE STREET,BATON ROUGE,70805,2724,EAST BATON ROUGE,2253547040,BR,CC,19267,R_11,0,R_11,3432,R_11,52,R_11,2012,00,00,00,-91.156007,30.496636,22,033,05000,441602,12,3,1023,2202,0,1,12940,0 -VA,VA0014,005,VA0014-005,PORTER BRANCH,2001 PARKWAY BOULEVARD,STAFFORD,22554,3972,STAFFORD,5406594909,BR,NO,23000,R_11,0,R_11,-1,U_11,-1,U_11,2012,00,00,00,-77.447472,38.464856,51,179,00000,132246,21,102.1399994,2007,5101,0,1,47900,0 -WI,WI0198,002,WI0198,MILTON PUBLIC LIBRARY,"430 E. HIGH ST., #100",MILTON,53563,1579,ROCK,6088687462,CE,NC,8000,R_11,0,R_11,2756,R_11,52,R_11,2012,00,00,00,-88.944389,42.77298,55,105,52200,160067,23,31,4016,5501,0,1,27500,0 -IA,IA0544,002,IA0563,WASHTA PUBLIC LIBRARY,100 S 5TH AVE,WASHTA,51061,1016,CHEROKEE,7124476546,CE,NO,1353,R_11,0,R_11,1222,R_11,52,R_11,2012,00,00,15,-95.720114,42.57533,19,035,82380,12029,43,803,2316,1904,0,1,0,. -NH,NH0174,002,NH895,GILMAN LIBRARY,100 MAIN STREET,ALTON,03809,4618,BELKNAP,6038752550,CE,NO,6996,R_11,0,R_11,1872,R_11,52,R_11,2012,00,00,00,-71.213211,43.452041,33,001,00980,60189,42,9665,2060,3301,0,1,29060,1 -CT,CT0175,002,15900,WETHERSFIELD PUBLIC LIBRARY,515 SILAS DEANE HIGHWAY,WETHERSFIELD,06109,2216,HARTFORD,8605292665,CE,NC,32877,R_11,0,R_11,2968,R_11,52,R_11,2012,00,00,00,-72.663363,41.711653,09,003,84970,896248,21,4922,1007,0901,0,1,25540,0 -NH,NH0030,002,NH89126,MAXFIELD PUBLIC LIBRARY,8 ROUTE 129,LOUDON,03307,0814,MERRIMACK,6037985153,CE,NO,6800,R_11,0,R_11,1768,R_11,52,R_11,2012,00,00,00,-71.467819,43.286141,33,013,43300,146614,42,360,2046,3302,0,1,18180,1 -NY,NY0005,015,0800000000,NORTH PARK BRANCH LIBRARY,975 HERTEL AVENUE,BUFFALO,14216,2201,ERIE,7168753748,BR,CC,3967,R_11,0,R_11,1568,R_11,52,R_11,2012,00,00,00,-78.871968,42.94775,36,029,11000,919627,11,50,1005,3626,0,1,15380,0 -OK,OK0018,002,24,CHANDLER PUBLIC LIBRARY,1021 S MANVEL,CHANDLER,74834,3853,LINCOLN,4052583204,CE,NO,2500,R_11,0,R_11,2496,R_11,52,R_11,2012,00,00,00,-96.880778,35.700289,40,081,13500,34293,32,9617,4057,4003,0,1,36420,0 -ND,ND0037,002,ND0037-002,HANKINSON PUBLIC LIBRARY,315 MAIN AVENUE S,HANKINSON,58041,0244,RICHLAND,7012427929,CE,NO,400,R_11,0,R_11,1198,R_11,51,R_11,2012,00,00,00,-96.902081,46.069547,38,077,34900,16261,43,9714,2037,3800,0,1,47420,1 -IN,IN0207,009,8412,TRAFALGAR BRANCH,424 TOWER STREET,TRAFALGAR,46181,8876,JOHNSON,3178789560,BR,NO,19000,R_11,0,R_11,3172,R_11,52,R_11,2012,00,00,00,-86.148687,39.413997,18,081,76310,141439,41,6114,2051,1809,0,1,26900,0 -KY,KY0071,003,MARION,MARION COUNTY PUBLIC LIBRARY,201 EAST MAIN STREET,LEBANON,40033,1133,MARION,2706924698,BS,NO,-3,U_11,1,R_11,1012,R_11,44,R_11,2012,00,00,00,-85.250194,37.570737,21,155,44344,20049,32,9702,2017,2101,0,1,0,. -NY,NY0562,033,6800310000,LEFRAK CITY,98-30 57 AVENUE,CORONA,11368,4696,QUEENS,7185927677,BR,CC,18000,R_11,0,R_11,1820,R_11,52,R_11,2012,00,00,00,-73.861226,40.738002,36,081,51000,2256438,11,455,3000,3614,4,15,35620,0 -NC,NC0014,003,R-PETTIGREW-P,PERQUIMANS COUNTY LIBRARY,110 W ACADEMY ST,HERTFORD,27944,1306,PERQUIMANS,2524265319,BR,NO,5320,R_11,0,R_11,2278,R_11,52,R_11,2012,00,00,15,-76.471096,36.188915,37,143,30900,13458,42,9202.009766,3019,3701,0,1,21020,1 -IL,IL0140,002,3022800,DIVERNON TOWNSHIP LIBRARY,221 SOUTH SECOND STREET,DIVERNON,62530,0140,SANGAMON,2176283813,CE,NO,1300,R_11,0,R_11,1560,R_11,-1,U_11,2012,00,00,00,-89.65852,39.565293,17,167,20045,198949,41,33,4122,1718,0,1,44100,0 -GA,GA0010,005,6282PICKENS,PICKENS COUNTY PUBLIC LIBRARY,100 LIBRARY LANE,JASPER,30143,1364,PICKENS,7066925411,BR,NO,11000,R_11,0,R_11,3320,R_11,52,R_11,2012,00,00,00,-84.420188,34.467478,13,227,41932,29431,32,502,1040,1309,2,14,12060,0 -MN,MN0085,002,S1310,BLOOMING PRAIRIE BRANCH LIBRARY,138 HIGHWAY AVE SOUTH,BLOOMING PRAIRIE,55917,0187,STEELE,5075837750,BR,NO,3000,R_11,0,R_11,2213,R_11,52,R_11,2012,00,00,00,-93.049227,43.866202,27,147,06580,36541,43,9608,4004,2701,0,5,36940,1 -MD,MD0003,039,MD0003,SOUTHEAST ANCHOR BRANCH,3601 EASTERN AVENUE,BALTIMORE,21224,M,BALTIMORE CITY,4103961580,BR,CC,27000,R_11,0,R_11,2527,R_11,52,R_11,2012,00,00,00,-76.567069,39.286587,24,510,04000,620216,11,2609,1010,2403,0,1,12580,0 -TX,TX0012,002,109,DENTON PUBLIC LIBRARY,502 OAKLAND ST,DENTON,76201,3102,DENTON,9403498750,CE,NC,22876,R_11,0,R_11,3161,R_11,52,R_11,2012,00,00,00,-97.129918,33.219673,48,121,19972,685601,12,206.0099945,3025,4826,0,1,19100,0 -MT,MT0044,002,747,LAUREL PUBLIC LIBRARY,720 WEST 3RD ST,LAUREL,59044,0068,YELLOWSTONE,4066284961,CE,NO,6000,R_11,0,R_11,2338,R_11,51,R_11,2012,00,00,00,-108.780888,45.671408,30,111,42700,149907,31,19.01000023,2007,3000,0,1,13740,0 -NJ,NJ0194,008,1225F,ISELIN BRANCH,1081 GREEN ST.,ISELIN,08830,2171,MIDDLESEX,7327267073,BR,NC,9000,R_11,0,R_11,-1,U_11,52,R_11,2012,00,00,00,-74.313243,40.571154,34,023,34470,816618,21,26.03000069,2000,3406,0,1,35620,0 -FL,FL0005,003,FL0005-003,ALAFAYA BRANCH,12000 EAST COLONIAL DRIVE,ORLANDO,32826,4705,ORANGE,4078357323,BR,CC,12000,R_11,0,R_11,3150,R_11,52,R_11,2012,00,00,00,-81.203356,28.566308,12,095,00410,1171018,21,167.2700043,1000,1209,0,1,36740,0 -LA,LA0002,003,10,MONTGOMERY BRANCH,940 CADDO STREET,MONTGOMERY,71454,0157,GRANT,3186463660,BR,NO,770,R_11,0,R_11,1456,R_11,52,R_11,2012,00,00,00,-92.889812,31.667523,22,043,51620,22115,42,202,1127,2205,0,1,10780,0 -IA,IA0400,002,VT,CRESCO PUBLIC LIBRARY,320 N ELM ST,CRESCO,52136,1452,HOWARD,5635472540,CE,NO,7800,R_11,0,R_11,3120,R_11,52,R_11,2012,00,00,15,-92.11626,43.374078,19,089,17220,9533,33,9601,4021,1901,0,1,0,. -IL,IL0031,002,3003100,BARCLAY PUBLIC LIBRARY DISTRICT,220 SOUTH MAIN STREET,WARRENSBURG,62573,0349,MACON,2176723621,CE,NO,5000,R_11,0,R_11,2600,R_11,-1,U_11,2012,00,00,00,-89.062322,39.929914,17,115,78916,110616,41,28,1137,1713,0,1,19500,0 -CT,CT0022,002,2100,DAVID M. HUNT LIBRARY,63 MAIN ST.,CANAAN,06031,0217,LITCHFIELD,8608247424,CE,NO,3600,R_11,0,R_11,1456,R_11,52,R_11,2012,00,00,00,-73.328793,42.026425,09,005,10870,188923,42,2602,1020,0905,0,1,45860,1 -CA,CA0119,002,M715,CROWELL PUBLIC LIBRARY,1890 HUNTINGTON DR.,SAN MARINO,91108,2595,LOS ANGELES,6263000777,CE,NC,29000,R_11,0,R_11,3224,R_11,52,R_11,2012,00,00,00,-118.112501,34.119348,06,037,68224,9889025,21,4641,4005,0627,0,1,31080,0 -MN,MN0043,009,M0715,NEW BRIGHTON LIBRARY,400 10TH STREET NW,NEW BRIGHTON,55112,6806,RAMSEY,6517246002,BR,NC,6250,R_11,0,R_11,294,R_11,9,R_11,2012,02,00,00,-93.191665,45.065114,27,123,45430,515336,21,412,1003,2704,0,1,33460,0 -GA,GA0008,018,GA0008,TYBEE,403 BUTLER AVENUE,TYBEE ISLAND,31328,9719,CHATHAM,9127867733,BR,NC,4000,R_11,0,R_11,1528,R_11,52,R_11,2012,00,00,00,-80.842618,32.008883,13,051,78036,271896,31,111.0299988,2022,1301,0,1,42340,0 -NH,NH0042,002,NH89137,WADLEIGH MEMORIAL LIBRARY,49 NASHUA STREET,MILFORD,03055,3753,HILLSBOROUGH,6036732408,CE,NO,13500,R_11,0,R_11,2923,R_11,52,R_11,2012,00,00,00,-71.648234,42.835511,33,011,47940,401842,22,162.0099945,5021,3302,0,1,31700,0 -NY,NY0069,002,7800020610,FREE LIBRARY OF THE BELMONT LITERARY AND HIST,2 WILLETS AVENUE,BELMONT,14813,1025,ALLEGANY,5852685308,CE,NO,8000,R_11,0,R_11,1000,R_11,50,R_11,2012,00,00,00,-78.034633,42.223293,36,003,05815,48744,42,9506,3060,3623,0,1,0,. -IL,IL0243,002,3034900,HIGHWOOD PUBLIC LIBRARY,102 HIGHWOOD AVENUE,HIGHWOOD,60040,1520,LAKE,8474325404,CE,NC,10363,R_11,0,R_11,2128,R_11,-1,U_11,2012,00,00,00,-87.810707,42.201979,17,097,34865,701575,21,8652,1007,1710,0,1,16980,0 -VA,VA0005,008,VA0005-008,WESTOVER BRANCH LIBRARY,1644 N.MCKINLEY STREET,ARLINGTON,22205,2855,ARLINGTON,7032285260,BR,NC,16403,R_11,0,R_11,-1,U_11,-1,U_11,2012,00,00,00,-77.140244,38.888422,51,013,03000,216118,12,1010,4003,5108,4,15,47900,0 -AZ,AZ0111,002,CE-COPPER,COPPER QUEEN LIBRARY,6 MAIN STREET,BISBEE,85603,9901,COCHISE,5204324232,CE,NO,9000,R_11,0,R_11,2236,R_11,52,R_11,2012,00,00,00,-109.915161,31.441616,04,003,06260,132770,33,10,2015,0402,0,1,43420,0 -SD,SD0042,002,SD0042-002,DOROTHEE PIKE MEMORIAL LIBRARY,225 N MAIN AVE,LAKE PRESTON,57249,0036,KINGSBURY,6058474843,CE,NO,392,R_11,0,R_11,988,R_11,52,R_11,2012,00,00,15,-97.377319,44.362653,46,077,35500,5152,43,9581,1191,4600,0,1,0,. -WI,WI0322,002,WI0322,TOMAHAWK PUBLIC LIBRARY,300 W. LINCOLN AVE.,TOMAHAWK,54487,1202,LINCOLN,7154532455,CE,NO,11465,R_11,0,R_11,2596,R_11,52,R_11,2012,00,00,00,-89.733724,45.473925,55,069,80125,28492,32,9602,2040,5507,0,1,32980,1 -NY,NY0146,002,2400113890,PECK MEMORIAL LIBRARY,24 MAIN STREET,MARATHON,13803,0325,CORTLAND,6078496135,CE,NO,3052,R_11,0,R_11,1490,R_11,52,R_11,2012,00,00,00,-76.033345,42.441233,36,023,45392,49619,42,9711,3064,3622,0,1,18660,1 -DC,DC0001,017,SOW,SOUTHWEST BRANCH LIBRARY,"900 WESLEY PLACE, SW",WASHINGTON,20024,4212,DIST OF COLUMBIA,2027244752,BR,CC,22032,R_11,0,R_11,2496,R_11,52,R_11,2012,00,00,00,-77.016219,38.879185,11,001,50000,619020,11,102,2044,1198,0,1,47900,0 -OR,OR0140,001,OR0140,LANE LIBRARY DISTRICT,64 W. OREGON AVE.,CRESWELL,97426,0366,LANE,5038953053,CE,NC,4950,R_11,0,R_11,2300,R_11,52,R_11,2012,00,00,00,-123.019529,43.917861,41,039,16950,353481,31,11.01000023,3086,4104,0,1,21660,0 -IL,IL0445,004,3056601,YORK STREET BRANCH LIBRARY,639 YORK STREET,QUINCY,62301,3918,ADAMS,2172231309,BR,NO,2775,R_11,0,R_11,2020,R_11,52,R_11,2012,00,00,00,-91.40595,39.929353,17,001,62367,67169,33,7,2054,1718,0,1,39500,1 -OH,OH0226,003,326B01,BROOKFIELD BRANCH LIBRARY,7032 GROVE ST,BROOKFIELD,44403,0052,TRUMBULL,3304488134,BR,NC,4200,R_11,0,R_11,2652,R_11,52,R_11,2012,00,00,00,-80.565232,41.232504,39,155,09200,208874,21,9314,4006,3913,0,1,49660,0 -WI,WI0052,002,WI0052,CAMPBELLSPORT PUBLIC LIBRARY,220 N. HELENA,CAMPBELLSPORT,53010,0405,FOND DU LAC,9205338534,CE,NC,10440,R_11,0,R_11,2020,R_11,52,R_11,2012,00,00,00,-88.276351,43.599297,55,039,12325,101856,42,422,3028,5506,0,1,22540,0 -AR,AR0001,017,AR0001-13,OLEY E. ROOKER LIBRARY,11 OTTER CREEK CT.,LITTLE ROCK,72210,M,PULASKI,5019075991,BR,NC,13550,R_11,0,R_11,-1,U_11,52,R_11,2012,00,00,00,-92.427498,34.659253,05,119,41000,386822,41,42.20000076,3009,0502,0,1,30780,0 -GA,GA0022,042,GA0022,NORTHEAST-SPRUILL OAKS REGIONAL LIBRARY,9560 SPRUILL ROAD,ALPHARETTA,30022,5550,FULTON,7703608820,BR,NC,25000,R_11,0,R_11,2882,R_11,52,R_11,2012,00,00,00,-84.22258,34.01515,13,121,42425,949580,21,114.25,2005,1306,0,1,12060,0 -ME,ME0247,002,93,NORTH GORHAM PUBLIC LIBRARY,2 STANDISH NECK ROAD,GORHAM,04038,2469,CUMBERLAND,2078922575,CE,NO,1200,R_11,0,R_11,750,R_11,52,R_11,2012,00,00,00,-70.45554,43.798963,23,005,00000,282669,32,41,2006,2301,0,1,38860,0 -ME,ME0079,002,83,FORT KENT PUBLIC LIBRARY,1 MONUMENT SQUARE,FORT KENT,04743,1216,AROOSTOOK,2078343048,CE,NO,3153,R_11,0,R_11,1579,R_11,52,R_11,2012,00,00,00,-68.595746,47.250002,23,003,25720,71388,43,9506,3020,2302,4,14,0,. -MI,MI0336,002,MI0336-002,THOMAS TOWNSHIP LIBRARY,8207 SHIELDS DRIVE,SAGINAW,48609,4814,SAGINAW,9897813770,CE,CC,14360,R_11,0,R_11,2732,R_11,52,R_11,2012,00,00,00,-84.074739,43.41882,26,145,73560,198990,22,120.0299988,1011,2604,0,1,40980,0 -TN,TN0133,006,KN001-05,FARRAGUT BRANCH LIBRARY,417 N. CAMPBELL STATION ROAD,KNOXVILLE,37934,2708,KNOX,8657771750,BR,CC,10100,R_11,0,R_11,2860,R_11,52,R_11,2012,00,00,00,-84.168446,35.88913,47,093,25760,436877,21,58.09000015,2020,4702,0,1,28940,0 -TX,TX0160,002,245,RED WALLER COMMUNITY LIBRARY,109 MELTON ST,MALAKOFF,75148,9347,HENDERSON,9034891818,CE,NO,4800,R_11,0,R_11,1500,R_11,50,R_11,2012,00,00,00,-96.014132,32.170392,48,213,46224,78753,33,9510,2059,4805,0,1,11980,1 -LA,LA0058,008,LA0058,MILTON H. LATTER MEMORIAL BRANCH,5120 ST. CHARLES AVENUE,NEW ORLEANS,70115,4941,ORLEANS,5045962625,BR,CC,9000,R_11,0,R_11,2704,R_11,52,R_11,2012,00,00,00,-90.109987,29.926997,22,071,55000,360341,11,108,1015,2202,0,1,35380,0 -WV,WV0020,005,019C,BOOKMOBILE,3100 EMERSON AVE.,PARKERSBURG,26104,2414,WOOD,3044204587,BS,NC,-3,U_11,1,R_11,1040,R_11,52,R_11,2012,00,00,00,-81.53833,39.28578,54,107,62140,86837,13,3,2003,5401,0,1,37620,0 -CA,CA0019,004,M615,CARNEGIE TECHNOLOGY CENTER BRANCH,420 HEBER AVE.,CALEXICO,92231,2559,IMPERIAL,7603575525,BR,NC,4000,R_11,0,R_11,708,R_11,52,R_11,2012,00,00,00,-115.493508,32.669062,06,025,09710,175897,31,121,3011,0651,0,1,20940,0 -PA,PA0385,026,926510006,LAWNCREST BRANCH LIBRARY,6098 RISING SUN AVENUE,PHILADELPHIA,19111,6009,PHILADELPHIA,2156850549,BR,CC,7184,R_11,0,R_11,-1,U_11,-1,U_11,2012,00,00,00,-75.099695,40.045973,42,101,60000,1538567,11,305.019989,1004,4213,0,1,37980,0 -NY,NY0306,002,3600414840,C. W. CLARK MEMORIAL LIBRARY,160 NORTH MAIN STREET,ORISKANY FALLS,13425,0670,ONEIDA,3158217850,CE,NC,3141,R_11,0,R_11,2060,R_11,52,R_11,2012,00,00,00,-75.459544,42.940144,36,065,55376,234155,42,256,2068,3622,0,1,46540,0 -NC,NC0061,005,C-UNION-W,WAXHAW LIBRARY,509 S PROVIDENCE ST,WAXHAW,28173,9339,UNION,7048433131,BR,NC,2900,R_11,0,R_11,1489,R_11,52,R_11,2012,00,00,15,-80.741342,34.922191,37,179,71460,205193,41,210.1399994,1009,3709,0,1,16740,0 -IL,IL0194,002,3029200,FRANKLIN PARK PUBLIC LIBRARY DISTRICT,10311 GRAND AVENUE,FRANKLIN PARK,60131,2225,COOK,8474556016,CE,NC,25000,R_11,0,R_11,3424,R_11,-1,U_11,2012,00,00,00,-87.882502,41.930234,17,031,27702,5214098,21,8114.02002,2009,1705,0,1,16980,0 -OH,OH0116,004,216B02,BALTIMORE BRANCH LIBRARY,205 E. MARKET ST.,BALTIMORE,43105,1326,FAIRFIELD,7408628505,BR,NC,3866,R_11,0,R_11,1632,R_11,51,R_11,2012,00,00,00,-82.600057,39.84531,39,045,03758,147355,31,303,3030,3915,0,1,18140,0 -MI,MI0360,002,MI0360-002,WATERVLIET DISTRICT LIBRARY,333 NORTH MAIN STREET,WATERVLIET,49098,9750,BERRIEN,2694636382,CE,NC,10272,R_11,0,R_11,2340,R_11,52,R_11,2012,00,00,00,-86.259586,42.19054,26,021,84500,156489,31,103,4027,2606,0,1,35660,0 -AL,AL0108,019,AL0108-019,BIRMINGHAM PUBLIC - TITUSVILLE BRANCH,26TH AVENUE SW,BIRMINGHAM,35211,2909,JEFFERSON,2053221140,BR,CC,9000,R_11,0,R_11,2385,R_11,52,R_11,2012,00,00,00,-86.833206,33.491538,01,073,07000,658967,12,51.00999832,3026,0107,4,15,13820,0 -TX,TX0443,005,86.66,SOUTH REGIONAL LIBRARY,2101 LAKE ROBBINS DR,THE WOODLANDS,77380,M,MONTGOMERY,9364427727,BR,NC,30000,R_11,0,R_11,3141,R_11,52,R_11,2012,00,00,00,-95.465652,30.162471,48,339,72656,471704,23,6917,1033,4808,0,1,26420,0 -FL,FL0012,042,FL0012-042,TAMARAC BRANCH,8701 W. COMMERCIAL BLVD.,TAMARAC,33351,4310,BROWARD,9547202282,BR,NC,30000,R_11,0,R_11,2672,R_11,52,R_11,2012,00,00,00,-80.267013,26.193663,12,011,39550,1784956,21,601.1400146,1004,1220,0,1,33100,0 -PA,PA0007,002,901300753,EVA K BOWLBY PUBLIC LIBRARY,311 NORTH WEST ST,WAYNESBURG,15370,1238,GREENE,7246279776,CE,NO,9791,R_11,0,R_11,2401,R_11,52,R_11,2012,00,00,00,-80.191465,39.899297,42,059,81832,38424,32,9706,3034,4218,0,1,0,. -IN,IN0068,012,2713,CENTRE TOWNSHIP BRANCH,1150 EAST KERN ROAD,SOUTH BEND,46614,6472,ST JOSEPH,5742513700,BR,CC,17160,R_11,0,R_11,2496,R_11,52,R_11,2012,00,00,00,-86.232966,41.607649,18,141,71000,266661,12,119,2011,1802,0,1,43780,0 -MI,MI0284,002,MI0284-002,REED CITY PUBLIC LIBRARY,410 WEST UPTON AVENUE,REED CITY,49677,1152,OSCEOLA,2318322131,CE,NO,3649,R_11,0,R_11,1872,R_11,52,R_11,2012,00,00,00,-85.516822,43.875995,26,133,67820,23455,33,9706,2012,2604,0,1,0,. -VT,VT0042,002,CORNWALL,CORNWALL FREE PUBLIC,2629 RT 30,CORNWALL,05753,9299,ADDISON,8024623615,CE,NO,315,R_11,0,R_11,1664,R_11,52,R_11,2012,00,00,00,-73.209627,43.961155,50,001,00000,36785,41,9609,1014,5000,0,1,0,. -TX,TX0065,002,157,LEE PUBLIC LIBRARY,312 W PACIFIC AVE,GLADEWATER,75647,2135,GREGG,9038452640,CE,NC,5500,R_11,0,R_11,2048,R_11,52,R_11,2012,00,00,00,-94.946153,32.535873,48,183,29660,122325,31,102,3008,4801,0,1,30980,0 -CT,CT0105,002,9500,PUBLIC LIBRARY OF NEW LONDON,63 HUNTINGTON ST.,NEW LONDON,06320,6194,NEW LONDON,8604471411,CE,CC,22000,R_11,0,R_11,2678,R_11,52,R_11,2012,00,00,00,-72.099541,41.354992,09,011,52280,274091,13,6905,2026,0902,0,1,35980,0 -NJ,NJ0195,009,1300G,MARLBORO LIBRARY,1 LIBRARY COURT,MARLBORO,07746,1102,MONMOUTH,7325369406,BR,NC,18100,R_11,0,R_11,-1,U_11,52,R_11,2012,00,00,00,-74.262612,40.330578,34,025,63900,630092,21,8097.009766,2021,3406,0,1,35620,0 -IL,IL0037,002,3003700,BEDFORD PARK PUBLIC LIBRARY DISTRICT,7816 WEST 65TH PLACE,BEDFORD PARK,60501,1914,COOK,7084586826,CE,NC,17000,R_11,0,R_11,3016,R_11,-1,U_11,2012,00,00,00,-87.816069,41.772696,17,031,04572,5214098,21,8205.009766,1026,1703,0,1,16980,0 -KS,KS0191,002,KS0059,CONWAY SPRINGS CITY LIBRARY,210 W. SPRINGS AVE,CONWAY SPRINGS,67031,0183,SUMNER,6204562859,CE,NO,980,R_11,0,R_11,1040,R_11,52,R_11,2012,00,00,00,-97.63682,37.390358,20,191,15325,23835,42,9622,3026,2004,0,1,48620,0 -NH,NH0055,002,NH89149,NEWFIELDS PUBLIC LIBRARY,76 MAIN STREET,NEWFIELDS,03856,0200,ROCKINGHAM,6037788169,CE,NO,-1,U_11,0,R_11,1456,R_11,52,R_11,2012,00,00,00,-70.932892,43.041363,33,015,00000,296085,31,675.0100098,2014,3301,0,1,14460,0 -WI,WI0308,002,WI0308,JEAN M. THOMSEN MEMORIAL LIBRARY,105 N. GERSHWIN ST.,STETSONVILLE,54480,0099,TAYLOR,7156782892,CE,NO,3990,R_11,0,R_11,1772,R_11,52,R_11,2012,00,00,00,-90.311907,45.077181,55,119,77100,20759,42,9606,3046,5507,0,1,0,. -FL,FL0011,019,FL0011-019,SUNTREE/VIERA PUBLIC LIBRARY,902 JORDAN BLASS DRIVE,MELBOURNE,32940,3350,BREVARD,3212554404,BR,NO,15000,R_11,0,R_11,2716,R_11,-1,U_11,2012,00,00,00,-80.679796,28.215339,12,009,00000,544322,21,641.2700195,4005,1208,0,1,37340,0 -NY,NY0071,002,7800021050,ESSENTIAL CLUB FREE LIBRARY,11 PRATT STREET,CANASERAGA,14822,9728,ALLEGANY,6075456443,CE,NO,2500,R_11,0,R_11,1144,R_11,52,R_11,2012,00,00,00,-77.775745,42.461616,36,003,12177,48744,42,9501,1037,3623,0,1,0,. -WI,WI0134,002,WI0134,HILLSBORO PUBLIC LIBRARY,819 HIGH AVE.,HILLSBORO,54634,0468,VERNON,6084892192,CE,NO,6800,R_11,0,R_11,1716,R_11,52,R_11,2012,00,00,00,-90.341447,43.652202,55,123,34825,30043,43,9601,2032,5503,0,1,0,. -FL,FL0021,002,FL0021-002,COLUMBIA COUNTY PUBLIC LIBRARY,308 N.W. COLUMBIA AVE.,LAKE CITY,32055,2891,COLUMBIA,3867581018,CE,NO,15400,R_11,0,R_11,2942,R_11,52,R_11,2012,00,00,00,-82.63798,30.192204,12,023,37775,67383,33,1105,2048,1203,0,1,29380,1 -FL,FL0001,005,FL0001-005,BELLEVIEW PUBLIC LIBRARY,13145 SE HWY 484,BELLEVIEW,34420,5831,MARION,3524382500,BR,NC,16000,R_11,0,R_11,2614,R_11,52,R_11,2012,00,00,00,-82.065647,29.028237,12,083,00000,332472,23,11.02000046,2051,1211,0,1,36100,0 -TN,TN0135,015,NA001-14,MADISON BRANCH LIBRARY,610 GALLATIN PIKE SOUTH,MADISON,37115,2192,DAVIDSON,6158625868,BR,CC,20680,R_11,0,R_11,2600,R_11,52,R_11,2012,00,00,00,-86.71406,36.257013,47,037,52006,635592,11,107.0100021,4001,4705,0,1,34980,0 -NH,NH0220,002,NH8995,WOODSVILLE FREE PUBLIC LIBRARY,14 SCHOOL LANE,WOODSVILLE,03785,1227,GRAFTON,6037473483,CE,NO,1360,IP10,0,R_11,1092,R_11,52,R_11,2012,00,00,00,-72.036754,44.150391,33,009,87140,88954,43,9606,3035,3302,0,1,17200,1 -NY,NY0651,002,8000580910,BROOKHAVEN FREE,273 BEAVER DAM RD,BROOKHAVEN,11719,9607,SUFFOLK,6312861923,CE,NC,5047,R_11,0,R_11,3140,R_11,52,R_11,2012,00,00,00,-72.911466,40.773631,36,103,09000,1500338,21,1592.040039,3015,3601,0,1,35620,0 -IL,IL0480,002,3060300,SCHILLER PARK PUBLIC LIBRARY,4200 OLD RIVER ROAD,SCHILLER PARK,60176,1630,COOK,8476780433,CE,NC,12500,R_11,0,R_11,3328,R_11,-1,U_11,2012,00,00,00,-87.860692,41.955696,17,031,68081,5214098,21,7708,1039,1705,0,1,16980,0 -NY,NY0004,019,0400300000,CROWN HEIGHTS,560 NEW YORK AVENUE,BROOKLYN,11225,5296,KINGS,7187731180,BR,CC,9280,R_11,0,R_11,2392,R_11,52,R_11,2012,00,00,00,-73.947776,40.661073,36,047,51000,2540230,11,804,1001,3609,0,1,35620,0 -OH,OH0039,011,139BK,STARK COUNTY DISTRICT LIBRARY,715 MARKET AVE.,CANTON,44702,1080,STARK,3304520665,BS,CC,-3,U_11,5,R_11,5976,R_11,52,R_11,2012,00,00,00,-81.373216,40.803018,39,151,12000,374328,13,7001,1048,3907,4,16,15940,0 -GA,GA0036,008,GA0036,MARION COUNTY PUBLIC LIBRARY,123 EAST 5TH AVE.,BUENNA VISTA,31803,0391,MARION,2296496385,BR,NO,4311,R_11,0,R_11,1588,R_11,52,R_11,2012,00,00,00,-84.522819,32.318596,13,197,11728,8706,42,9202,3058,1302,0,1,17980,0 -NJ,NJ0195,006,1300D,HAZLET MEMORIAL LIBRAY,251 MIDDLE ROAD,HAZLET,07730,1904,MONMOUTH,7322647164,BR,NC,10200,R_11,0,R_11,-1,U_11,52,R_11,2012,00,00,00,-74.176972,40.427274,34,025,00000,630092,21,8021,5017,3406,0,1,35620,0 -SC,SC8004,006,SC8004-002,NINETY-SIX BRANCH LIBRARY,100 SOUTH CAMBRIDGE STREET,NINETY-SIX,29666,1111,GREENWOOD,8645434749,BR,NO,3750,R_11,0,R_11,2314,R_11,52,R_11,2012,00,00,00,-82.023766,34.175093,45,047,50290,69739,42,9707.019531,4019,4503,0,1,24940,1 -WI,WI0382,002,WI0382,DRESSER PUBLIC LIBRARY,117 S. CENTRAL AVE.,DRESSER,54009,0547,POLK,7157552944,CE,NO,1274,R_11,0,R_11,1250,R_11,50,R_11,2012,00,00,00,-92.633499,45.355307,55,095,20850,43984,42,9607,3074,5507,0,1,0,. -OK,OK0061,002,65,MAYSVILLE PUBLIC LIBRARY,508 WILLIAMS,MAYSVILLE,73057,0878,GARVIN,4058674748,CE,NO,3125,R_11,0,R_11,2184,R_11,52,R_11,2012,00,00,00,-97.40843,34.817807,40,049,47150,27387,42,6816,1114,4004,0,1,0,. -CA,CA0027,006,M623,ATLANTIC BRANCH,2269 S. ATLANTIC BLVD,COMMERCE,90040,3955,LOS ANGELES,3237801176,BR,NC,3852,R_11,0,R_11,2080,R_11,52,R_11,2012,00,00,00,-118.164562,34.005171,06,037,14974,9889025,21,5323.02002,4024,0640,0,1,31080,0 -NJ,NJ0114,005,0705C,FRANKLIN BRANCH,192 DODD STREET,EAST ORANGE,07017,2107,ESSEX,9732667053,BR,NC,500,R_11,0,R_11,-1,U_11,52,R_11,2012,00,00,00,-74.21016,40.781824,34,013,19390,786754,21,100,1001,3410,0,1,35620,0 -OH,OH0177,002,277C,MORLEY LIBRARY,184 PHELPS ST.,PAINESVILLE,44077,3926,LAKE,4403523383,CE,NC,17500,R_11,0,R_11,3951,R_11,52,R_11,2012,00,00,00,-81.243767,41.727571,39,085,59416,229873,21,2045,1008,3914,0,1,17460,0 -NY,NY0420,002,5600221120,CARTHAGE FREE LIBRARY,412 BUDD STREET,CARTHAGE,13619,1206,JEFFERSON,3154932620,CE,NO,4800,R_11,0,R_11,2392,R_11,52,R_11,2012,00,00,00,-75.609727,43.979609,36,045,12683,118163,33,609,3023,3621,0,1,48060,0 -KY,KY0073,003,MARTIN,RUFUS M. REED PUBLIC LIBRARY,1442 RIVERFRONT ROAD,LOVELY,41231,0359,MARTIN,6063956500,BR,NO,3400,R_11,0,R_11,2080,R_11,52,R_11,2012,00,00,00,-82.367849,37.78319,21,159,00000,12839,42,9503,1003,2105,X,14,0,. -MT,MT0051,005,1216,FRENCHTOWN SCHOOL AND COMMUNITY LIBRARY (MISSOULA CO.),16495 MAIN STREET,FRENCHTOWN,59834,M,MISSOULA,4066262730,BR,NO,4500,R_11,0,R_11,1040,R_11,52,R_11,2012,00,14,00,-114.227048,47.011982,30,063,29350,110114,42,16,1029,3000,0,1,33540,0 -PA,PA0147,002,908561265,MARY S BIESECKER PUB LIBRARY,230 S ROSINA AVE,SOMERSET,15501,1937,SOMERSET,8144454011,CE,NC,3531,R_11,0,R_11,2345,R_11,52,R_11,2012,00,00,00,-79.082912,40.00724,42,111,71776,77382,32,210,3002,4212,0,1,43740,1 -IA,IA0251,002,MM,ALGONA PUBLIC LIBRARY,210 N PHILLIPS,ALGONA,50511,1735,KOSSUTH,5152955476,CE,NO,16000,R_11,0,R_11,2912,R_11,52,R_11,2012,00,00,15,-94.226822,43.070044,19,109,01135,15369,33,9505,2042,1904,0,1,0,. -NY,NY0004,061,0400300000,WINDSOR TERRACE,160 EAST 5TH STREET,BROOKLYN,11218,1702,KINGS,7186869707,BR,CC,7500,R_11,0,R_11,2392,R_11,52,R_11,2012,00,00,00,-73.97657,40.648829,36,047,51000,2540230,11,500,2004,3609,0,1,35620,0 -IN,IN0177,002,7284,BLOOMFIELD-EASTERN GREENE COUNTY PUBLIC LIBRARY,125 SOUTH FRANKLIN STREET,BLOOMFIELD,47424,1406,GREENE,8123844125,CE,NO,13400,R_11,0,R_11,2808,R_11,52,R_11,2012,00,00,00,-86.93852,39.02559,18,055,05716,33061,32,9554,4026,1808,0,1,0,. -NJ,NJ0223,002,NJ0223-002,BUTLER PUBLIC LIBRARY,1 ACE ROAD,BUTLER,07045,1375,MORRIS,2018383262,CE,NC,6324,R_11,0,R_11,2659,R_11,52,R_11,2012,00,00,00,-74.342983,41.003358,34,027,09040,495941,21,405,1007,3411,0,1,35620,0 -TX,TX0247,011,329.299,CINCO RANCH BRANCH LIBRARY,2620 COMMERCIAL CENTER BLVD,KATY,77494,6407,FORT BEND,2813951311,BR,NC,33500,R_11,0,R_11,2524,R_11,52,R_11,2012,00,00,00,-95.787502,29.725205,48,157,00000,607952,41,6731.02002,1089,4822,2,15,26420,0 -MN,MN0035,015,M0213,ANOKA: RUM RIVER,4201 6TH AVE NW,ANOKA,55303,M,ANOKA,7635764695,BR,NC,30150,R_11,0,R_11,2312,R_11,52,R_11,2012,00,00,15,-93.380862,45.22981,27,003,01720,333226,21,505.0499878,3007,2706,0,1,33460,0 -OH,OH0002,019,102B17,HIGHLAND SQUARE BRANCH LIBRARY,807 W MARKET ST,AKRON,44303,1010,SUMMIT,3303762927,BR,CC,11500,R_11,0,R_11,2964,R_11,52,R_11,2012,00,06,15,-81.543313,41.096492,39,153,01000,541227,12,5073,3001,3913,0,1,10420,0 -MA,MA0015,002,ATHOL,ATHOL PUBLIC LIBRARY,568 MAIN STEET,ATHOL,01331,1888,WORCESTER,9782499515,CE,NO,8000,R_11,0,R_11,2350,R_11,52,R_11,2012,00,00,15,-72.228447,42.593893,25,027,02515,803785,32,7031,1026,2502,4,14,49340,0 -CA,CA0105,022,M701,SOUTHGATE LIBRARY,6132 66TH AVE.,SACRAMENTO,95823,2706,SACRAMENTO,9162642700,BR,CC,12000,R_11,0,R_11,1980,R_11,49,R_11,2012,00,00,00,-121.433101,38.492443,06,067,24498,1436262,21,50.02000046,1004,0607,0,1,40900,0 -CT,CT0124,002,11000,PLAINVILLE PUBLIC LIBRARY,56 EAST MAIN ST.,PLAINVILLE,06062,1934,HARTFORD,8607931446,CE,NC,30000,R_11,0,R_11,3016,R_11,52,R_11,2012,00,00,00,-72.865298,41.671845,09,003,00000,896248,21,4205,4005,0905,0,1,25540,0 -TX,TX0069,002,160,LIBRARY OF GRAHAM,910 CHERRY ST,GRAHAM,76450,3547,YOUNG,9405490600,CE,NO,14000,R_11,0,R_11,2184,R_11,52,R_11,2012,00,00,00,-98.587919,33.099243,48,503,30392,18385,33,9505,2005,4819,0,1,0,. -MI,MI0322,002,MI0322-002,SAINT CLAIR SHORES PUBLIC LIBRARY,22500 ELEVEN MILE ROAD,SAINT CLAIR SHORES,48081,1399,MACOMB,5867719020,CE,NC,30000,R_11,0,R_11,3092,R_11,52,R_11,2012,00,00,00,-82.887251,42.496575,26,099,70760,842881,21,2511,1001,2609,0,1,19820,0 -TX,TX0183,002,269,WARD COUNTY LIBRARY,409 S DWIGHT ST,MONAHANS,79756,4609,WARD,4329433332,CE,NO,15000,R_11,0,R_11,2452,R_11,52,R_11,2012,00,00,00,-102.887829,31.59298,48,475,48936,10713,32,9502,1052,4823,0,5,0,. -AL,AL0248,001,506-003,SOUTHSIDE PUBLIC LIBRARY,2255 HIGHWAY 77,SOUTHSIDE,35907,7905,ETOWAH,2564420105,CE,NO,1000,R_11,0,R_11,1404,R_11,52,R_11,2012,00,00,00,-86.022239,33.918787,01,055,71832,104298,23,105.0100021,3044,0104,0,1,23460,0 -ME,ME0296,002,268,SWANS ISLAND PUBLIC LIBRARY,451 ATLANTIC ROAD,SWAN'S ISLAND,04685,0012,HANCOCK,2075264330,CE,NO,3500,R_11,0,R_11,1200,R_11,52,R_11,2012,00,00,00,-68.417378,44.174678,23,009,00000,54551,43,9662,2136,2302,0,1,0,. -KS,KS0200,002,KS0020,BELLE PLAINE PUBLIC LIBRARY,222 W. 5TH,BELLE PLAINE,67013,0700,SUMNER,6204883431,CE,NO,1987,R_11,0,R_11,1508,R_11,52,R_11,2012,00,00,00,-97.281473,37.393656,20,191,05500,23835,42,9621,4011,2004,0,1,48620,0 -CA,CA0144,006,M740,SUTTER BRANCH LIBRARY,2147 CALIFORNIA ST.,SUTTER,95982,2546,SUTTER,5307550485,BR,NC,970,R_11,0,R_11,996,R_11,52,R_11,2012,00,00,00,-121.748948,39.158979,06,101,77378,94951,23,508,2033,0603,0,1,49700,0 -DC,DC0001,012,NOE,NORTHEAST BRANCH LIBRARY,"330 7TH STREET, NE",WASHINGTON,20002,6104,DIST OF COLUMBIA,2026983320,BR,CC,13900,R_11,0,R_11,2496,R_11,52,R_11,2012,00,00,00,-76.996192,38.893985,11,001,50000,619020,11,83.01999664,1014,1198,0,1,47900,0 -KY,KY0109,007,WARREN,GRAHAM DRIVE COMMUNITY BRANCH,305 GRAHAM DRIVE,BOWLING GREEN,42101,1140,WARREN,2707811441,BR,CC,1627,R_11,0,R_11,2496,R_11,52,R_11,2012,00,00,00,-86.441796,37.003255,21,227,08902,115491,13,102,3031,2102,0,1,14540,0 -FL,FL0025,015,FL0025-015,KENDALL BRANCH LIBRARY,9101 S.W. 97 AVENUE,MIAMI,33176,1985,MIAMI-DADE,3052790520,BR,CC,14000,R_11,0,R_11,2559,R_11,52,R_11,2012,00,00,00,-80.350799,25.685623,12,086,36100,2565440,13,84.05000305,1009,1227,0,1,33100,0 -PA,PA0532,002,902024075,SENIOR BUS,1301 BEAVER AVENUE,PITTSBURGH,15233,2342,ALLEGHENY,4123211853,BS,CC,-3,U_11,1,R_11,403,R_11,50,R_11,2012,00,00,00,-80.028932,40.453027,42,003,61000,1227442,11,9806,1011,4214,0,1,38300,0 -NJ,NJ0204,002,NJ0204-002,FAIR HAVEN PUBLIC LIBRARY,748 RIVER ROAD,FAIR HAVEN,07704,3397,MONMOUTH,2017475031,CE,NC,1750,R_11,0,R_11,-1,U_11,52,R_11,2012,00,00,00,-74.035853,40.364273,34,025,22440,630092,21,8037,5000,3404,0,1,35620,0 -VI,VI0002,002,VI0002-002,ELAINE I. SPRAUVE PUBLIC LIBRARY,EST. ENIGHED,CRUZ BAY,00830,M,ST. JOHN,3407766359,BR,NO,1200,R_11,0,R_11,-1,U_11,-1,U_11,2012,00,00,00,-64.740289,18.338647,78,020,00000,-1,41,9501,3092,7898,X,14,0,. -NC,NC0104,001,C-CHATHAM-BS,CHATHAM COUNTY BOOKMOBILE,500 N 2ND AVE,SILER CITY,27344,3123,CHATHAM,9195427725,BS,NC,-3,U_11,1,R_11,1000,R_11,50,R_11,2012,00,14,15,-79.46469,35.727411,37,037,61860,65280,32,204.0200043,1049,3702,0,1,20500,0 -TX,TX0402,003,49.362,SOUTHMOST BRANCH LIBRARY,4320 SOUTHMOST BLVD,BROWNSVILLE,78521,M,CAMERON,9565481055,BR,CC,20000,R_11,0,R_11,2602,R_11,52,R_11,2012,00,00,00,-97.449958,25.896555,48,061,10768,412577,12,133.0599976,1008,4834,0,1,15180,0 -OH,OH0163,008,OH0163-008,BUCKEYE LAKE BRANCH,41 W FIRST ST,BUCKEYE LAKE,43008,1708,LICKING,7409280472,BR,NO,1200,R_11,0,R_11,2433,R_11,52,R_11,2012,00,00,00,-82.482686,39.933065,39,089,09890,167194,23,7583,3012,3912,0,1,18140,0 -GA,GA0023,006,6632HILTON,IDA HILTON PUBLIC LIBRARY,1105 NORTHWAY,DARIEN,31305,1227,MCINTOSH,9124272124,BR,NO,6000,R_11,0,R_11,2110,R_11,-1,U_11,2012,00,00,00,-81.417136,31.394249,13,191,00000,14229,31,1102,1079,1301,X,14,15260,0 -LA,LA0057,015,60,ROSEDALE BRANCH,4036 JEFFERSON HIGHWAY,OLD JEFFERSON,70121,1630,JEFFERSON,5048384350,BR,NC,7138,R_11,0,R_11,2496,R_11,52,R_11,2012,00,00,00,-90.168164,29.961761,22,051,38145,433523,21,245,1006,2201,0,1,35380,0 -WI,WI0059,002,WI0059,CHIPPEWA FALLS PUBLIC LIBRARY,105 W. CENTRAL ST.,CHIPPEWA FALLS,54729,2397,CHIPPEWA,7157231146,CE,NC,18025,R_11,0,R_11,2682,R_11,52,R_11,2012,00,00,00,-91.393911,44.935456,55,017,14575,62857,23,103,1021,5503,0,1,20740,0 -WI,WI0065,002,WI0065,COLFAX PUBLIC LIBRARY,613 MAIN ST.,COLFAX,54730,0525,DUNN,7159624334,CE,NO,1800,R_11,0,R_11,1820,R_11,52,R_11,2012,00,00,00,-91.727195,44.999765,55,033,16275,44021,42,9703,3057,5503,0,1,32860,1 -DC,DC0001,016,SOE,SOUTHEAST BRANCH LIBRARY,"403 7TH STREET, SE",WASHINGTON,20003,2700,DIST OF COLUMBIA,2026983377,BR,CC,6431,R_11,0,R_11,2496,R_11,52,R_11,2012,00,00,00,-76.996189,38.88387,11,001,50000,619020,11,65,2000,1198,0,1,47900,0 -MI,MI0376,002,MI0376-002,GEORGE W. SPINDLER MEMORIAL LIBRARY,186 NORTH MAIN STREET,WOODLAND,48897,0068,BARRY,2693674694,CE,NO,1738,R_11,0,R_11,1040,R_11,52,R_11,2012,00,00,00,-85.13415,42.728953,26,015,88420,58936,42,101,2030,2603,0,1,24340,0 -WV,WV0040,002,39,CLARKSBURG-HARRISON CO. PUBLIC LIBRARY,404 W. PIKE STREET,CLARKSBURG,26301,2712,HARRISON,3046272236,CE,NO,30442,R_11,0,R_11,3484,R_11,52,R_11,2012,00,00,00,-80.340592,39.280797,54,033,15628,69298,32,301,1055,5401,0,1,17220,1 -IL,IL0524,002,3064900,THREE RIVERS PUBLIC LIBRARY DISTRICT,25207 WEST CHANNON DRIVE,CHANNAHON,60410,0300,WILL,8154676200,CE,NC,9200,R_11,0,R_11,3464,R_11,-1,U_11,2012,00,00,00,-88.224651,41.434087,17,197,12476,680584,21,8833.049805,1044,1716,0,1,16980,0 -IA,IA0048,002,BW,BERTHA BARTLETT PUBLIC LIBRARY,503 BROAD ST,STORY CITY,50248,1133,STORY,5157332685,CE,NO,8000,R_11,0,R_11,2738,R_11,52,R_11,2012,00,00,15,-93.597291,42.187095,19,169,75675,90816,31,106,1023,1904,0,1,11180,0 -MT,MT0011,003,1634,BUTTE-SILVER BOW PUBLIC LIBRARY SOUTH BRANCH,3100 HARRISON AVE,BUTTE,59701,M,SILVER BOW,4067233361,BR,NO,1200,R_11,0,R_11,1008,R_11,28,R_11,2012,02,00,00,-112.508,45.979868,30,093,11397,34362,33,7,1022,3000,0,1,15580,1 -NM,NM0110,001,NM0110-001,PUEBLO DE ABIQUIU LIBRARY & CULTURAL CENTER,COUNTY RD. #187 BUILDING #29,ABIQUIU,87510,0838,RIO ARRIBA,5056854884,CE,NO,400,R_11,0,R_11,1329,R_11,52,R_11,2012,00,00,00,-106.231605,36.156557,35,039,00000,40353,42,4,2205,3503,X,14,21580,1 -OR,OR0072,002,OR0072,SCAPPOOSE PUBLIC LIBRARY,52469 SE 2ND ST,SCAPPOOSE,97056,0400,COLUMBIA,5035437123,CE,NC,11020,R_11,0,R_11,2190,R_11,52,R_11,2012,00,00,00,-122.876339,45.756527,41,009,65500,49357,32,9709,2004,4101,0,1,38900,0 -TN,TN0132,005,CH001-04,EASTGATE BRANCH LIBRARY,5705 MARLIN ROAD,CHATTANOOGA,37411,5508,HAMILTON,4237575310,BR,CC,9084,R_11,0,R_11,2340,R_11,52,R_11,2012,00,00,00,-85.211325,35.007946,47,065,14000,340870,12,30,2044,4703,0,1,16860,0 -UT,UT0013,002,CACHCO1501,RICHMOND PUBLIC LIBRARY,38 WEST MAIN STREET,RICHMOND,84333,1409,CACHE,4352585525,CE,NO,2700,R_11,0,R_11,1565,R_11,52,R_11,2012,00,00,00,-111.809843,41.922642,49,005,63680,114559,41,1.00999999,3061,4901,0,1,30860,0 -IN,IN0123,002,5092,LOGANSPORT-CASS COUNTY PUBLIC LIBRARY,616 EAST BROADWAY,LOGANSPORT,46947,3155,CASS,5747536383,CE,NO,40448,R_11,0,R_11,3900,R_11,52,R_11,2012,00,00,00,-86.362725,40.754552,18,017,44658,38810,32,9515,2020,1804,0,1,30900,1 -WV,WV0082,002,81,GREENBRIER COUNTY PUBLIC LIBRARY,152 ROBERT W. MCCORMICK DRIVE,LEWISBURG,24901,1312,GREENBRIER,3046477568,CE,NO,13500,R_11,0,R_11,3068,R_11,52,R_11,2012,00,00,00,-80.448962,37.803713,54,025,46636,35710,33,9506,1041,5403,4,14,0,. -ME,ME0042,002,43,CARIBOU PUBLIC LIBRARY,30 HIGH STREET,CARIBOU,04736,2796,AROOSTOOK,2074934214,CE,NO,11000,R_11,0,R_11,2548,R_11,52,R_11,2012,00,00,00,-68.010616,46.860615,23,003,10565,71388,33,9513,3015,2302,0,1,0,. -MA,MA0091,002,ERVING,ERVING PUBLIC LIBRARY,17 MOORE ST.,ERVING,01344,9717,FRANKLIN,4134233348,CE,NO,1865,R_11,0,R_11,1044,R_11,52,R_11,2012,00,00,00,-72.490752,42.584079,25,011,41410,71646,32,404,2075,2502,0,1,24640,1 -CA,CA0068,026,M664,GEORGE SOUTH MERCED BRANCH,401 LESHER DRIVE,MERCED,95340,6572,MERCED,2097253909,BR,CC,1200,R_11,0,R_11,1300,R_11,52,R_11,2012,00,00,00,-120.504007,37.293699,06,047,46898,259966,13,15.02999973,1021,0616,0,1,32900,0 -MI,MI0240,012,MI0240-012,NORTH MUSKEGON BRANCH LIBRARY,1522 RUDDIMAN DRIVE,NORTH MUSKEGON,49445,3038,MUSKEGON,2317446080,BR,NC,3700,R_11,0,R_11,2496,R_11,52,R_11,2012,00,00,00,-86.267321,43.256368,26,121,58640,170021,22,15,2009,2602,0,1,34740,0 -VA,VA0076,002,VA0076-002,SHENANDOAH COUNTY LIBRARY,514 STONEY CREEK BLVD.,EDINBURG,22824,9142,SHENANDOAH,5409848200,CE,NO,13000,R_11,0,R_11,-1,U_11,-1,U_11,2012,00,00,00,-78.569018,38.825883,51,171,25008,42270,42,406,2037,5106,0,1,0,. -NJ,NJ0188,002,NJ0188-002,SAYREVILLE FREE PUBLIC LIBRARY,1050 WASHINGTON ROAD,PARLIN,08859,1061,MIDDLESEX,2017270212,CE,NC,19500,R_11,0,R_11,3172,R_11,52,R_11,2012,00,00,00,-74.315701,40.464356,34,023,65790,816618,21,73.01000214,2000,3406,0,1,35620,0 -NY,NY0251,002,3200482455,ALICE CURTIS DESMOND AND HAMILTON FISH LIBRARY,472 ROUTE 403,GARRISON,10524,9803,PUTNAM,8454243020,CE,NC,9467,R_11,0,R_11,2258,R_11,52,R_11,2012,00,00,00,-73.94238,41.375377,36,079,00000,99916,41,108,2025,3618,0,1,35620,0 -GA,GA0004,015,GA0004,GORDON PUBLIC LIBRARY,284 MILLEDGEVILLE ROAD,GORDON,31031,0336,WILKINSON,4786285352,BR,NO,3385,R_11,0,R_11,2524,R_11,52,R_11,2012,00,00,00,-83.32934,32.882452,13,319,33980,9453,42,9603,3018,1308,0,1,0,. -NJ,NJ0284,006,1900D,SUSSEX-WANTAGE BRANCH,69 ROUTE 639,SUSSEX,07461,2301,SUSSEX,9738753940,BR,NC,1400,R_11,0,R_11,-1,U_11,52,R_11,2012,00,00,00,-74.608678,41.210574,34,037,71670,148517,32,3712,1004,3405,4,16,35620,0 -NE,NE0038,002,040C,BRUNSWICK PUBLIC LIBRARY,303 FRANKLIN ST.,BRUNSWICK,68720,0011,ANTELOPE,4028422105,CE,NO,810,R_11,0,R_11,832,R_11,52,R_11,2012,00,00,00,-97.97071,42.292652,31,003,00000,6631,43,9796,2236,3103,0,1,0,. -KS,KS0155,002,KS0111,GOODLAND PUBLIC LIBRARY,812 BROADWAY,GOODLAND,67735,3037,SHERMAN,7858995461,CE,NO,13500,R_11,0,R_11,2810,R_11,52,R_11,2012,00,00,00,-101.710437,39.351002,20,181,26875,6039,33,4537,4002,2001,0,1,0,. -MI,MI0361,019,MI0361-002,WAYNE COUNTY PUBLIC LIBRARY,30555 MICHIGAN AVENUE,WESTLAND,48186,5310,WAYNE,7347277310,CE,NC,7000,R_11,0,R_11,2172,R_11,52,R_11,2012,00,00,00,-83.340555,42.286454,26,163,86000,1801789,21,5687,4017,2613,0,1,19820,0 -TX,TX0583,002,577,DILLEY PUBLIC LIBRARY,231 W FM 117,DILLEY,78017,3503,FRIO,8309651951,CE,NO,2340,R_11,0,R_11,2000,R_11,50,R_11,2012,00,00,00,-99.170073,28.67396,48,163,20428,17367,33,9503,1253,4823,0,1,0,. -GA,GA0022,026,GA0022,PEACHTREE BRANCH LIBRARY,"1315 PEACHTREE STREET, N.E.",ATLANTA,30309,7515,FULTON,4048857830,BR,CC,10000,R_11,0,R_11,1806,R_11,52,R_11,2012,00,00,00,-84.38501,33.790479,13,121,04000,949580,11,4,1013,1305,0,1,12060,0 -IN,IN0180,003,7531,ELLETTSVILLE BRANCH,600 WEST TEMPERANCE,ELLETTSVILLE,47429,1324,MONROE,8123492515,BR,NC,12000,R_11,0,R_11,3328,R_11,52,R_11,2012,00,00,00,-86.625077,39.234002,18,105,20800,140063,23,13.02999973,1050,1809,0,1,14020,0 -WI,WI0023,002,WI0023,BELLEVILLE PUBLIC LIBRARY,130 S. VINE ST.,BELLEVILLE,53508,0140,DANE,6084241812,CE,NC,4600,R_11,0,R_11,2652,R_11,52,R_11,2012,00,00,00,-89.53399,42.858436,55,025,06300,496456,42,126,3121,5502,0,1,31540,0 -CA,CA0146,004,M742,EL RETIRO BRANCH LIBRARY,126 VISTA DEL PARQUE,REDONDO BEACH,90277,6112,LOS ANGELES,3103750922,BR,NC,4158,R_11,0,R_11,2222,R_11,52,R_11,2012,00,00,00,-118.379754,33.815298,06,037,80000,9889025,12,6513.040039,2005,0633,0,1,31080,0 -TX,TX0099,040,189.360,MCGOVERN-STELLA LINK BRANCH LIBRARY,7405 STELLA LINK RD,HOUSTON,77025,M,HARRIS,8323932630,BR,CC,20393,R_11,0,R_11,2098,R_11,50,R_11,2012,00,00,00,-95.440039,29.698416,48,201,35000,4173695,11,4130,2005,4807,0,1,26420,0 diff --git a/examples/pig/test/populate-cli.txt b/examples/pig/test/populate-cli.txt deleted file mode 100644 index b2dda585e2..0000000000 --- a/examples/pig/test/populate-cli.txt +++ /dev/null @@ -1,134 +0,0 @@ -create keyspace PigTest with placement_strategy = 'org.apache.cassandra.locator.SimpleStrategy' and strategy_options={replication_factor:1}; -use PigTest; -create column family SomeApp with -key_validation_class = UTF8Type and -default_validation_class = LexicalUUIDType and -comparator = UTF8Type and -column_metadata = -[ - {column_name: name, validation_class: UTF8Type, index_type: KEYS}, - {column_name: vote_type, validation_class: UTF8Type}, - {column_name: rating, validation_class: Int32Type}, - {column_name: score, validation_class: LongType}, - {column_name: percent, validation_class: FloatType}, - {column_name: atomic_weight, validation_class: DoubleType}, - {column_name: created, validation_class: DateType}, -]; - -create column family CopyOfSomeApp with -key_validation_class = UTF8Type and -default_validation_class = LexicalUUIDType and -comparator = UTF8Type and -column_metadata = -[ - {column_name: name, validation_class: UTF8Type, index_type: KEYS}, - {column_name: vote_type, validation_class: UTF8Type}, - {column_name: rating, validation_class: Int32Type}, - {column_name: score, validation_class: LongType}, - {column_name: percent, validation_class: FloatType}, - {column_name: atomic_weight, validation_class: DoubleType}, - {column_name: created, validation_class: DateType}, -]; - -set SomeApp['foo']['name'] = 'User Foo'; -set SomeApp['foo']['vote_type'] = 'like'; -set SomeApp['foo']['rating'] = 8; -set SomeApp['foo']['score'] = 125000; -set SomeApp['foo']['percent'] = '85.0'; -set SomeApp['foo']['atomic_weight'] = '2.7182818284590451'; -set SomeApp['foo']['created'] = 1335890877; - -set SomeApp['bar']['name'] = 'User Bar'; -set SomeApp['bar']['vote_type'] = 'like'; -set SomeApp['bar']['rating'] = 9; -set SomeApp['bar']['score'] = 15000; -set SomeApp['bar']['percent'] = '35.0'; -set SomeApp['bar']['atomic_weight'] = '3.1415926535897931'; -set SomeApp['bar']['created'] = 1335890877; - -set SomeApp['baz']['name'] = 'User Baz'; -set SomeApp['baz']['vote_type'] = 'dislike'; -set SomeApp['baz']['rating'] = 3; -set SomeApp['baz']['score'] = 512000; -set SomeApp['baz']['percent'] = '95.3'; -set SomeApp['baz']['atomic_weight'] = '1.61803399'; -set SomeApp['baz']['created'] = 1335890877; -set SomeApp['baz']['extra1'] = lexicaluuid(); -set SomeApp['baz']['extra2'] = lexicaluuid(); -set SomeApp['baz']['extra3'] = lexicaluuid(); - -set SomeApp['qux']['name'] = 'User Qux'; -set SomeApp['qux']['vote_type'] = 'dislike'; -set SomeApp['qux']['rating'] = 2; -set SomeApp['qux']['score'] = 12000; -set SomeApp['qux']['percent'] = '64.7'; -set SomeApp['qux']['atomic_weight'] = '0.660161815846869'; -set SomeApp['qux']['created'] = 1335890877; -set SomeApp['qux']['extra1'] = lexicaluuid(); -set SomeApp['qux']['extra2'] = lexicaluuid(); -set SomeApp['qux']['extra3'] = lexicaluuid(); -set SomeApp['qux']['extra4'] = lexicaluuid(); -set SomeApp['qux']['extra5'] = lexicaluuid(); -set SomeApp['qux']['extra6'] = lexicaluuid(); -set SomeApp['qux']['extra7'] = lexicaluuid(); - -create column family U8 with - key_validation_class = UTF8Type and - comparator = UTF8Type; - -create column family Bytes with - key_validation_class = BytesType and - comparator = UTF8Type; - -set U8['foo']['x'] = ascii('Z'); -set Bytes[ascii('foo')]['x'] = ascii('Z'); - -create column family CC with - key_validation_class = UTF8Type and - default_validation_class=CounterColumnType - and comparator=UTF8Type; - -incr CC['chuck']['kick']; -incr CC['chuck']['kick']; -incr CC['chuck']['kick']; -incr CC['chuck']['fist']; - -create column family Compo - with key_validation_class = UTF8Type - and default_validation_class = UTF8Type - and comparator = 'CompositeType(UTF8Type,UTF8Type)'; - -set Compo['punch']['bruce:lee'] = 'ouch'; -set Compo['punch']['bruce:bruce'] = 'hunh?'; -set Compo['kick']['bruce:lee'] = 'oww'; -set Compo['kick']['bruce:bruce'] = 'watch it, mate'; - -create column family CompoInt - with key_validation_class = UTF8Type - and default_validation_class = UTF8Type - and comparator = 'CompositeType(LongType,LongType)'; - -set CompoInt['clock']['1:0'] = 'z'; -set CompoInt['clock']['1:30'] = 'zzzz'; -set CompoInt['clock']['2:30'] = 'daddy?'; -set CompoInt['clock']['6:30'] = 'coffee...'; - -create column family CompoIntCopy - with key_validation_class = UTF8Type - and default_validation_class = UTF8Type - and comparator = 'CompositeType(LongType,LongType)'; - -create column family CompoKey - with key_validation_class = 'CompositeType(UTF8Type,LongType)' - and default_validation_class = UTF8Type - and comparator = LongType; - -set CompoKey['clock:10']['1'] = 'z'; -set CompoKey['clock:20']['1'] = 'zzzz'; -set CompoKey['clock:30']['2'] = 'daddy?'; -set CompoKey['clock:40']['6'] = 'coffee...'; - -create column family CompoKeyCopy - with key_validation_class = 'CompositeType(UTF8Type,LongType)' - and default_validation_class = UTF8Type - and comparator = LongType; diff --git a/examples/pig/test/populate-cql.txt b/examples/pig/test/populate-cql.txt deleted file mode 100644 index 00b81db317..0000000000 --- a/examples/pig/test/populate-cql.txt +++ /dev/null @@ -1,33 +0,0 @@ -CREATE KEYSPACE cql3ks WITH replication = {'class': 'SimpleStrategy', 'replication_factor': 1 }; -USE cql3ks; - -CREATE TABLE test (a int PRIMARY KEY, b int); -CREATE TABLE moredata (x int PRIMARY KEY, y int); -INSERT INTO test (a,b) VALUES (1,1); -INSERT INTO test (a,b) VALUES (2,2); -INSERT INTO test (a,b) VALUES (3,3); -INSERT INTO moredata (x, y) VALUES (4,4); -INSERT INTO moredata (x, y) VALUES (5,5); -INSERT INTO moredata (x, y) VALUES (6,6); - -CREATE TABLE compotable (a int, b int, c text, d text, PRIMARY KEY (a,b,c)); -INSERT INTO compotable (a, b , c , d ) VALUES ( 1,1,'One','match'); -INSERT INTO compotable (a, b , c , d ) VALUES ( 2,2,'Two','match'); -INSERT INTO compotable (a, b , c , d ) VALUES ( 3,3,'Three','match'); -INSERT INTO compotable (a, b , c , d ) VALUES ( 4,4,'Four','match'); - -create table compmore (id int PRIMARY KEY, x int, y int, z text, data text); -INSERT INTO compmore (id, x, y, z,data) VALUES (1,5,6,'Fix','nomatch'); -INSERT INTO compmore (id, x, y, z,data) VALUES (2,6,5,'Sive','nomatch'); -INSERT INTO compmore (id, x, y, z,data) VALUES (3,7,7,'Seven','match'); -INSERT INTO compmore (id, x, y, z,data) VALUES (4,8,8,'Eight','match'); -INSERT INTO compmore (id, x, y, z,data) VALUES (5,9,10,'Ninen','nomatch'); - - -CREATE TABLE collectiontable(m text PRIMARY KEY, n map); -UPDATE collectiontable SET n['key1'] = 'value1' WHERE m = 'book1'; -UPDATE collectiontable SET n['key2'] = 'value2' WHERE m = 'book2'; -UPDATE collectiontable SET n['key3'] = 'value3' WHERE m = 'book3'; -UPDATE collectiontable SET n['key4'] = 'value4' WHERE m = 'book4'; - - diff --git a/examples/pig/test/test_cql_storage.pig b/examples/pig/test/test_cql_storage.pig deleted file mode 100644 index 822748ef7e..0000000000 --- a/examples/pig/test/test_cql_storage.pig +++ /dev/null @@ -1,14 +0,0 @@ -moretestvalues= LOAD 'cql://cql3ks/moredata/' USING CqlNativeStorage; -insertformat= FOREACH moretestvalues GENERATE TOTUPLE(TOTUPLE('a',x)),TOTUPLE(y); -STORE insertformat INTO 'cql://cql3ks/test?output_query=UPDATE+cql3ks.test+set+b+%3D+%3F' USING CqlNativeStorage; - --- composite key -moredata = load 'cql://cql3ks/compmore' USING CqlNativeStorage; -insertformat = FOREACH moredata GENERATE TOTUPLE (TOTUPLE('a',x),TOTUPLE('b',y), TOTUPLE('c',z)),TOTUPLE(data); -STORE insertformat INTO 'cql://cql3ks/compotable?output_query=UPDATE%20cql3ks.compotable%20SET%20d%20%3D%20%3F' USING CqlNativeStorage; - --- collection column -collectiontable = LOAD 'cql://cql3ks/collectiontable/' USING CqlNativeStorage; --- recs= (((m,kk)),((map,(m,mm),(n,nn)))) -recs= FOREACH collectiontable GENERATE TOTUPLE(TOTUPLE('m', m) ), TOTUPLE(TOTUPLE('map', TOTUPLE('m', 'mm'), TOTUPLE('n', 'nn'))); -store recs INTO 'cql://cql3ks/collectiontable?output_query=update+cql3ks.collectiontable+set+n+%3D+%3F' USING CqlNativeStorage(); diff --git a/examples/pig/test/test_storage.pig b/examples/pig/test/test_storage.pig deleted file mode 100644 index 026cb025a9..0000000000 --- a/examples/pig/test/test_storage.pig +++ /dev/null @@ -1,85 +0,0 @@ -rows = LOAD 'cassandra://PigTest/SomeApp' USING CassandraStorage(); --- full copy -STORE rows INTO 'cassandra://PigTest/CopyOfSomeApp' USING CassandraStorage(); --- single tuple -onecol = FOREACH rows GENERATE key, percent; -STORE onecol INTO 'cassandra://PigTest/CopyOfSomeApp' USING CassandraStorage(); --- bag only -other = FOREACH rows GENERATE key, columns; -STORE other INTO 'cassandra://PigTest/CopyOfSomeApp' USING CassandraStorage(); - - --- filter -likes = FILTER rows by vote_type.value eq 'like' and rating.value > 5; -dislikes_extras = FILTER rows by vote_type.value eq 'dislike' AND COUNT(columns) > 0; - --- store these too -STORE likes INTO 'cassandra://PigTest/CopyOfSomeApp' USING CassandraStorage(); -STORE dislikes_extras INTO 'cassandra://PigTest/CopyOfSomeApp' USING CassandraStorage(); - --- filter to fully visible rows (no uuid columns) and dump -visible = FILTER rows BY COUNT(columns) == 0; -dump visible; - - - --- test key types with a join -U8 = load 'cassandra://PigTest/U8' using CassandraStorage(); -Bytes = load 'cassandra://PigTest/Bytes' using CassandraStorage(); - --- cast key to chararray -b = foreach Bytes generate (chararray)key, columns; - --- key in Bytes is a bytearray, U8 chararray -a = join Bytes by key, U8 by key; -dump a - --- key should now be cast into a chararray -c = join b by (chararray)key, U8 by (chararray)key; -dump c - - --- --- Test counter column family support --- -CC = load 'cassandra://PigTest/CC' using CassandraStorage(); - -total_hits = foreach CC generate key, SUM(columns.value); - -dump total_hits; - --- --- Test CompositeType --- - -compo = load 'cassandra://PigTest/Compo' using CassandraStorage(); - -compo = foreach compo generate key as method, flatten(columns); - -lee = filter compo by columns::name == ('bruce','lee'); - -dump lee; - -night = load 'cassandra://PigTest/CompoInt' using CassandraStorage(); -night = foreach night generate flatten(columns); -night = foreach night generate (int)columns::name.$0+(double)columns::name.$1/60 as hour, columns::value as noise; - --- What happens at the darkest hour? -darkest = filter night by hour > 2 and hour < 5; - -dump darkest; - -compo_int_rows = LOAD 'cassandra://PigTest/CompoInt' USING CassandraStorage(); -STORE compo_int_rows INTO 'cassandra://PigTest/CompoIntCopy' USING CassandraStorage(); - --- --- Test CompositeKey --- - -compokeys = load 'cassandra://PigTest/CompoKey' using CassandraStorage(); -compokeys = filter compokeys by key.$1 == 40; - -dump compokeys; - -compo_key_rows = LOAD 'cassandra://PigTest/CompoKey' USING CassandraStorage(); -STORE compo_key_rows INTO 'cassandra://PigTest/CompoKeyCopy' USING CassandraStorage(); diff --git a/src/java/org/apache/cassandra/hadoop/pig/CqlNativeStorage.java b/src/java/org/apache/cassandra/hadoop/pig/CqlNativeStorage.java deleted file mode 100644 index ba0a37d6dc..0000000000 --- a/src/java/org/apache/cassandra/hadoop/pig/CqlNativeStorage.java +++ /dev/null @@ -1,1082 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.apache.cassandra.hadoop.pig; - -import java.io.ByteArrayInputStream; -import java.io.ByteArrayOutputStream; -import java.io.IOException; -import java.io.ObjectInputStream; -import java.io.ObjectOutputStream; -import java.io.Serializable; -import java.io.UnsupportedEncodingException; -import java.net.URLDecoder; -import java.nio.ByteBuffer; -import java.util.*; - -import com.datastax.driver.core.ColumnMetadata; -import com.datastax.driver.core.Metadata; -import com.datastax.driver.core.Row; -import com.datastax.driver.core.Session; -import com.datastax.driver.core.TableMetadata; -import com.datastax.driver.core.exceptions.NoHostAvailableException; - -import org.apache.cassandra.db.marshal.*; -import org.apache.cassandra.exceptions.AuthenticationException; -import org.apache.cassandra.exceptions.ConfigurationException; -import org.apache.cassandra.exceptions.SyntaxException; -import org.apache.cassandra.hadoop.ConfigHelper; -import org.apache.cassandra.hadoop.HadoopCompat; -import org.apache.cassandra.hadoop.cql3.CqlBulkOutputFormat; -import org.apache.cassandra.hadoop.cql3.CqlBulkRecordWriter; -import org.apache.cassandra.hadoop.cql3.CqlConfigHelper; -import org.apache.cassandra.hadoop.cql3.CqlRecordReader; -import org.apache.cassandra.serializers.CollectionSerializer; -import org.apache.cassandra.utils.*; -import org.apache.hadoop.conf.Configuration; -import org.apache.hadoop.fs.Path; -import org.apache.hadoop.mapreduce.*; -import org.apache.pig.*; -import org.apache.pig.Expression.OpType; -import org.apache.pig.ResourceSchema.ResourceFieldSchema; -import org.apache.pig.backend.executionengine.ExecException; -import org.apache.pig.backend.hadoop.executionengine.mapReduceLayer.PigSplit; -import org.apache.pig.data.*; -import org.apache.pig.impl.util.UDFContext; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; -import org.yaml.snakeyaml.external.biz.base64Coder.Base64Coder; - -@Deprecated -public class CqlNativeStorage extends LoadFunc implements StoreFuncInterface, LoadMetadata -{ - protected String DEFAULT_INPUT_FORMAT; - protected String DEFAULT_OUTPUT_FORMAT; - - protected String username; - protected String password; - protected String keyspace; - protected String column_family; - protected String loadSignature; - protected String storeSignature; - - protected Configuration conf; - protected String inputFormatClass; - protected String outputFormatClass; - protected int splitSize = 64 * 1024; - protected String partitionerClass; - protected boolean usePartitionFilter = false; - protected String initHostAddress; - protected String rpcPort; - protected int nativeProtocolVersion = 1; - - private static final Logger logger = LoggerFactory.getLogger(CqlNativeStorage.class); - private static String BULK_OUTPUT_FORMAT = "org.apache.cassandra.hadoop.cql3.CqlBulkOutputFormat"; - private int pageSize = 1000; - private String columns; - private String outputQuery; - private String whereClause; - - private RecordReader reader; - private RecordWriter, List> writer; - private String nativePort; - private String nativeCoreConnections; - private String nativeMaxConnections; - private String nativeMaxSimultReqs; - private String nativeConnectionTimeout; - private String nativeReadConnectionTimeout; - private String nativeReceiveBufferSize; - private String nativeSendBufferSize; - private String nativeSolinger; - private String nativeTcpNodelay; - private String nativeReuseAddress; - private String nativeKeepAlive; - private String nativeAuthProvider; - private String nativeSSLTruststorePath; - private String nativeSSLKeystorePath; - private String nativeSSLTruststorePassword; - private String nativeSSLKeystorePassword; - private String nativeSSLCipherSuites; - private String inputCql; - - private boolean bulkOutputFormat = false; - private String bulkCfSchema; - private String bulkInsertStatement; - private String bulkOutputLocation; - private int bulkBuffSize = -1; - private int bulkStreamThrottle = -1; - private int bulkMaxFailedHosts = -1; - private boolean bulkDeleteSourceOnSuccess = true; - private String bulkTableAlias; - - public CqlNativeStorage() - { - this(1000); - } - - /** @param pageSize limit number of CQL rows to fetch in a thrift request */ - public CqlNativeStorage(int pageSize) - { - super(); - this.pageSize = pageSize; - DEFAULT_INPUT_FORMAT = "org.apache.cassandra.hadoop.cql3.CqlInputFormat"; - DEFAULT_OUTPUT_FORMAT = "org.apache.cassandra.hadoop.cql3.CqlOutputFormat"; - } - - public void prepareToRead(RecordReader reader, PigSplit split) - { - this.reader = reader; - if (reader instanceof CqlRecordReader) { - nativeProtocolVersion = ((CqlRecordReader) reader).getNativeProtocolVersion(); - } - } - - public void prepareToWrite(RecordWriter writer) - { - this.writer = writer; - } - - /** get next row */ - public Tuple getNext() throws IOException - { - try - { - // load the next pair - if (!reader.nextKeyValue()) - return null; - - TableInfo tableMetadata = getCfInfo(loadSignature); - Row row = reader.getCurrentValue(); - Tuple tuple = TupleFactory.getInstance().newTuple(tableMetadata.getColumns().size()); - Iterator itera = tableMetadata.getColumns().iterator(); - int i = 0; - while (itera.hasNext()) - { - ColumnInfo cdef = itera.next(); - ByteBuffer columnValue = row.getBytesUnsafe(cdef.getName()); - if (columnValue != null) - { - AbstractType validator = getValidatorMap(tableMetadata).get(ByteBufferUtil.bytes(cdef.getName())); - setTupleValue(tuple, i, cqlColumnToObj(ByteBufferUtil.bytes(cdef.getName()), columnValue, - tableMetadata), validator); - } - else - tuple.set(i, null); - i++; - } - return tuple; - } - catch (InterruptedException e) - { - throw new IOException(e.getMessage()); - } - } - - /** convert a cql column to an object */ - private Object cqlColumnToObj(ByteBuffer name, ByteBuffer columnValue, TableInfo cfDef) throws IOException - { - // standard - Map validators = getValidatorMap(cfDef); - return StorageHelper.cassandraToObj(validators.get(name), columnValue, nativeProtocolVersion); - } - - /** set the value to the position of the tuple */ - private void setTupleValue(Tuple tuple, int position, Object value, AbstractType validator) throws ExecException - { - if (validator instanceof CollectionType) - setCollectionTupleValues(tuple, position, value, validator); - else - StorageHelper.setTupleValue(tuple, position, value); - } - - /** set the values of set/list at and after the position of the tuple */ - private void setCollectionTupleValues(Tuple tuple, int position, Object value, AbstractType validator) throws ExecException - { - if (validator instanceof MapType) - { - setMapTupleValues(tuple, position, value, validator); - return; - } - AbstractType elementValidator; - if (validator instanceof SetType) - elementValidator = ((SetType) validator).getElementsType(); - else if (validator instanceof ListType) - elementValidator = ((ListType) validator).getElementsType(); - else - return; - - int i = 0; - Tuple innerTuple = TupleFactory.getInstance().newTuple(((Collection) value).size()); - for (Object entry : (Collection) value) - { - setTupleValue(innerTuple, i, cassandraToPigData(entry, elementValidator), elementValidator); - i++; - } - tuple.set(position, innerTuple); - } - - /** set the values of set/list at and after the position of the tuple */ - private void setMapTupleValues(Tuple tuple, int position, Object value, AbstractType validator) throws ExecException - { - AbstractType keyValidator = ((MapType) validator).getKeysType(); - AbstractType valueValidator = ((MapType) validator).getValuesType(); - - int i = 0; - Tuple innerTuple = TupleFactory.getInstance().newTuple(((Map) value).size()); - for(Map.Entry entry : ((Map)value).entrySet()) - { - Tuple mapEntryTuple = TupleFactory.getInstance().newTuple(2); - setTupleValue(mapEntryTuple, 0, cassandraToPigData(entry.getKey(), keyValidator), keyValidator); - setTupleValue(mapEntryTuple, 1, cassandraToPigData(entry.getValue(), valueValidator), valueValidator); - innerTuple.set(i, mapEntryTuple); - i++; - } - tuple.set(position, innerTuple); - } - - private Object cassandraToPigData(Object obj, AbstractType validator) - { - if (validator instanceof DecimalType || validator instanceof InetAddressType) - return validator.getString(validator.decompose(obj)); - return obj; - } - - /** get the columnfamily definition for the signature */ - protected TableInfo getCfInfo(String signature) throws IOException - { - UDFContext context = UDFContext.getUDFContext(); - Properties property = context.getUDFProperties(CqlNativeStorage.class); - TableInfo cfInfo; - try - { - cfInfo = cfdefFromString(property.getProperty(signature)); - } - catch (ClassNotFoundException e) - { - throw new IOException(e); - } - return cfInfo; - } - - /** return the CfInfo for the column family */ - protected TableMetadata getCfInfo(Session client) - throws NoHostAvailableException, - AuthenticationException, - IllegalStateException - { - // get CF meta data - return client.getCluster().getMetadata().getKeyspace(Metadata.quote(keyspace)).getTable(Metadata.quote(column_family)); - } - - /** convert key tuple to key map */ - private Map tupleToKeyMap(Tuple t) throws IOException - { - Map keys = new HashMap(); - for (int i = 0; i < t.size(); i++) - { - if (t.getType(i) != DataType.TUPLE) - throw new IOException("keys was not a tuple"); - Tuple inner = (Tuple) t.get(i); - if (inner.size() != 2) - throw new IOException("Keys were not in name and value pairs"); - Object name = inner.get(0); - if (name == null) - throw new IOException("Key name was empty"); - keys.put(name.toString(), objToBB(inner.get(1))); - } - return keys; - } - - /** convert object to ByteBuffer */ - protected ByteBuffer objToBB(Object o) - { - if (o == null) - return nullToBB(); - if (o instanceof java.lang.String) - return ByteBuffer.wrap(new DataByteArray((String)o).get()); - if (o instanceof Integer) - return Int32Type.instance.decompose((Integer)o); - if (o instanceof Long) - return LongType.instance.decompose((Long)o); - if (o instanceof Float) - return FloatType.instance.decompose((Float)o); - if (o instanceof Double) - return DoubleType.instance.decompose((Double)o); - if (o instanceof UUID) - return ByteBuffer.wrap(UUIDGen.decompose((UUID) o)); - if(o instanceof Tuple) { - List objects = ((Tuple)o).getAll(); - //collections - if (objects.size() > 0 && objects.get(0) instanceof String) - { - String collectionType = (String) objects.get(0); - if ("set".equalsIgnoreCase(collectionType) || - "list".equalsIgnoreCase(collectionType)) - return objToListOrSetBB(objects.subList(1, objects.size())); - else if ("map".equalsIgnoreCase(collectionType)) - return objToMapBB(objects.subList(1, objects.size())); - - } - return objToCompositeBB(objects); - } - - return ByteBuffer.wrap(((DataByteArray) o).get()); - } - - private ByteBuffer objToListOrSetBB(List objects) - { - List serialized = new ArrayList(objects.size()); - for(Object sub : objects) - { - ByteBuffer buffer = objToBB(sub); - serialized.add(buffer); - } - return CollectionSerializer.pack(serialized, objects.size(), 3); - } - - private ByteBuffer objToMapBB(List objects) - { - List serialized = new ArrayList(objects.size() * 2); - for(Object sub : objects) - { - List keyValue = ((Tuple)sub).getAll(); - for (Object entry: keyValue) - { - ByteBuffer buffer = objToBB(entry); - serialized.add(buffer); - } - } - return CollectionSerializer.pack(serialized, objects.size(), 3); - } - - private ByteBuffer objToCompositeBB(List objects) - { - List serialized = new ArrayList(objects.size()); - int totalLength = 0; - for(Object sub : objects) - { - ByteBuffer buffer = objToBB(sub); - serialized.add(buffer); - totalLength += 2 + buffer.remaining() + 1; - } - ByteBuffer out = ByteBuffer.allocate(totalLength); - for (ByteBuffer bb : serialized) - { - int length = bb.remaining(); - out.put((byte) ((length >> 8) & 0xFF)); - out.put((byte) (length & 0xFF)); - out.put(bb); - out.put((byte) 0); - } - out.flip(); - return out; - } - - /** send CQL query request using data from tuple */ - private void cqlQueryFromTuple(Map key, Tuple t, int offset) throws IOException - { - for (int i = offset; i < t.size(); i++) - { - if (t.getType(i) != DataType.TUPLE) - throw new IOException("Output type was not a tuple"); - - Tuple inner = (Tuple) t.get(i); - if (inner.size() > 0) - { - List bindedVariables = bindedVariablesFromTuple(inner); - if (bindedVariables.size() <= 0) - throw new IOException("Missing binded variables"); - sendCqlQuery(key, bindedVariables); - } - } - } - - /** compose a list of binded variables */ - private List bindedVariablesFromTuple(Tuple t) throws IOException - { - List variables = new ArrayList(); - for (int i = 0; i < t.size(); i++) - variables.add(objToBB(t.get(i))); - return variables; - } - - /** writer write the data by executing CQL query */ - private void sendCqlQuery(Map key, List bindedVariables) throws IOException - { - try - { - writer.write(key, bindedVariables); - } - catch (InterruptedException e) - { - throw new IOException(e); - } - } - - /** get the validators */ - protected Map getValidatorMap(TableInfo cfDef) throws IOException - { - Map validators = new HashMap<>(); - for (ColumnInfo cd : cfDef.getColumns()) - { - if (cd.getTypeName() != null) - { - try - { - AbstractType validator = TypeParser.parseCqlName(cd.getTypeName()); - if (validator instanceof CounterColumnType) - validator = LongType.instance; - validators.put(ByteBufferUtil.bytes(cd.getName()), validator); - } - catch (ConfigurationException | SyntaxException e) - { - throw new IOException(e); - } - } - } - return validators; - } - - /** schema: (value, value, value) where keys are in the front. */ - public ResourceSchema getSchema(String location, Job job) throws IOException - { - setLocation(location, job); - TableInfo cfInfo = getCfInfo(loadSignature); - // top-level schema, no type - ResourceSchema schema = new ResourceSchema(); - - // get default validators - Map validators = getValidatorMap(cfInfo); - - // will contain all fields for this schema - List allSchemaFields = new ArrayList(); - - for (ColumnInfo cdef : cfInfo.getColumns()) - { - ResourceFieldSchema valSchema = new ResourceFieldSchema(); - AbstractType validator = validators.get(ByteBufferUtil.bytes(cdef.getName())); - valSchema.setName(cdef.getName()); - valSchema.setType(StorageHelper.getPigType(validator)); - allSchemaFields.add(valSchema); - } - - // top level schema contains everything - schema.setFields(allSchemaFields.toArray(new ResourceFieldSchema[allSchemaFields.size()])); - return schema; - } - - public void setPartitionFilter(Expression partitionFilter) throws IOException - { - UDFContext context = UDFContext.getUDFContext(); - Properties property = context.getUDFProperties(CqlNativeStorage.class); - property.setProperty(StorageHelper.PARTITION_FILTER_SIGNATURE, partitionFilterToWhereClauseString(partitionFilter)); - } - - /** - * Return cql where clauses for the corresponding partition filter. Make sure the data format matches - * Only support the following Pig data types: int, long, float, double, boolean and chararray - * */ - private String partitionFilterToWhereClauseString(Expression expression) throws IOException - { - Expression.BinaryExpression be = (Expression.BinaryExpression) expression; - OpType op = expression.getOpType(); - String opString = op.toString(); - switch (op) - { - case OP_EQ: - opString = " = "; - case OP_GE: - case OP_GT: - case OP_LE: - case OP_LT: - String name = be.getLhs().toString(); - String value = be.getRhs().toString(); - return String.format("%s %s %s", name, opString, value); - case OP_AND: - return String.format("%s AND %s", partitionFilterToWhereClauseString(be.getLhs()), partitionFilterToWhereClauseString(be.getRhs())); - default: - throw new IOException("Unsupported expression type: " + opString); - } - } - - /** retrieve where clause for partition filter */ - private String getWhereClauseForPartitionFilter() - { - UDFContext context = UDFContext.getUDFContext(); - Properties property = context.getUDFProperties(CqlNativeStorage.class); - return property.getProperty(StorageHelper.PARTITION_FILTER_SIGNATURE); - } - - /** - * output: (((name, value), (name, value)), (value ... value), (value...value)) - * bulk output: ((value ... value), (value...value)) - * - * */ - public void putNext(Tuple t) throws IOException - { - if (t.size() < 1) - { - // simply nothing here, we can't even delete without a key - logger.warn("Empty output skipped, filter empty tuples to suppress this warning"); - return; - } - - if (t.getType(0) != DataType.TUPLE) - throw new IOException("First argument in output must be a tuple"); - - if (!bulkOutputFormat && t.getType(1) != DataType.TUPLE) - throw new IOException("Second argument in output must be a tuple"); - - if (bulkOutputFormat) - { - cqlQueryFromTuple(null, t, 0); - } - else - { - Map key = tupleToKeyMap((Tuple)t.get(0)); - cqlQueryFromTuple(key, t, 1); - } - } - - /** set read configuration settings */ - public void setLocation(String location, Job job) throws IOException - { - conf = job.getConfiguration(); - setLocationFromUri(location); - - if (username != null && password != null) - { - ConfigHelper.setInputKeyspaceUserNameAndPassword(conf, username, password); - CqlConfigHelper.setUserNameAndPassword(conf, username, password); - } - if (splitSize > 0) - ConfigHelper.setInputSplitSize(conf, splitSize); - if (partitionerClass!= null) - ConfigHelper.setInputPartitioner(conf, partitionerClass); - if (initHostAddress != null) - ConfigHelper.setInputInitialAddress(conf, initHostAddress); - if (rpcPort != null) - ConfigHelper.setInputRpcPort(conf, rpcPort); - if (nativePort != null) - CqlConfigHelper.setInputNativePort(conf, nativePort); - if (nativeCoreConnections != null) - CqlConfigHelper.setInputCoreConnections(conf, nativeCoreConnections); - if (nativeMaxConnections != null) - CqlConfigHelper.setInputMaxConnections(conf, nativeMaxConnections); - if (nativeMaxSimultReqs != null) - CqlConfigHelper.setInputMaxSimultReqPerConnections(conf, nativeMaxSimultReqs); - if (nativeConnectionTimeout != null) - CqlConfigHelper.setInputNativeConnectionTimeout(conf, nativeConnectionTimeout); - if (nativeReadConnectionTimeout != null) - CqlConfigHelper.setInputNativeReadConnectionTimeout(conf, nativeReadConnectionTimeout); - if (nativeReceiveBufferSize != null) - CqlConfigHelper.setInputNativeReceiveBufferSize(conf, nativeReceiveBufferSize); - if (nativeSendBufferSize != null) - CqlConfigHelper.setInputNativeSendBufferSize(conf, nativeSendBufferSize); - if (nativeSolinger != null) - CqlConfigHelper.setInputNativeSolinger(conf, nativeSolinger); - if (nativeTcpNodelay != null) - CqlConfigHelper.setInputNativeTcpNodelay(conf, nativeTcpNodelay); - if (nativeReuseAddress != null) - CqlConfigHelper.setInputNativeReuseAddress(conf, nativeReuseAddress); - if (nativeKeepAlive != null) - CqlConfigHelper.setInputNativeKeepAlive(conf, nativeKeepAlive); - if (nativeAuthProvider != null) - CqlConfigHelper.setInputNativeAuthProvider(conf, nativeAuthProvider); - if (nativeSSLTruststorePath != null) - CqlConfigHelper.setInputNativeSSLTruststorePath(conf, nativeSSLTruststorePath); - if (nativeSSLKeystorePath != null) - CqlConfigHelper.setInputNativeSSLKeystorePath(conf, nativeSSLKeystorePath); - if (nativeSSLTruststorePassword != null) - CqlConfigHelper.setInputNativeSSLTruststorePassword(conf, nativeSSLTruststorePassword); - if (nativeSSLKeystorePassword != null) - CqlConfigHelper.setInputNativeSSLKeystorePassword(conf, nativeSSLKeystorePassword); - if (nativeSSLCipherSuites != null) - CqlConfigHelper.setInputNativeSSLCipherSuites(conf, nativeSSLCipherSuites); - - ConfigHelper.setInputColumnFamily(conf, keyspace, column_family); - setConnectionInformation(); - - CqlConfigHelper.setInputCQLPageRowSize(conf, String.valueOf(pageSize)); - if (inputCql != null) - CqlConfigHelper.setInputCql(conf, inputCql); - if (columns != null) - CqlConfigHelper.setInputColumns(conf, columns); - if (whereClause != null) - CqlConfigHelper.setInputWhereClauses(conf, whereClause); - - String whereClauseForPartitionFilter = getWhereClauseForPartitionFilter(); - String wc = whereClause != null && !whereClause.trim().isEmpty() - ? whereClauseForPartitionFilter == null ? whereClause: String.format("%s AND %s", whereClause.trim(), whereClauseForPartitionFilter) - : whereClauseForPartitionFilter; - - if (wc != null) - { - logger.trace("where clause: {}", wc); - CqlConfigHelper.setInputWhereClauses(conf, wc); - } - if (System.getenv(StorageHelper.PIG_INPUT_SPLIT_SIZE) != null) - { - try - { - ConfigHelper.setInputSplitSize(conf, Integer.parseInt(System.getenv(StorageHelper.PIG_INPUT_SPLIT_SIZE))); - } - catch (NumberFormatException e) - { - throw new IOException("PIG_INPUT_SPLIT_SIZE is not a number", e); - } - } - - if (ConfigHelper.getInputInitialAddress(conf) == null) - throw new IOException("PIG_INPUT_INITIAL_ADDRESS or PIG_INITIAL_ADDRESS environment variable not set"); - if (ConfigHelper.getInputPartitioner(conf) == null) - throw new IOException("PIG_INPUT_PARTITIONER or PIG_PARTITIONER environment variable not set"); - if (loadSignature == null) - loadSignature = location; - - initSchema(loadSignature); - } - - /** set store configuration settings */ - public void setStoreLocation(String location, Job job) throws IOException - { - conf = HadoopCompat.getConfiguration(job); - setLocationFromUri(location); - - if (username != null && password != null) - ConfigHelper.setOutputKeyspaceUserNameAndPassword(conf, username, password); - if (splitSize > 0) - ConfigHelper.setInputSplitSize(conf, splitSize); - if (partitionerClass!= null) - ConfigHelper.setOutputPartitioner(conf, partitionerClass); - if (rpcPort != null) - { - ConfigHelper.setOutputRpcPort(conf, rpcPort); - ConfigHelper.setInputRpcPort(conf, rpcPort); - } - if (initHostAddress != null) - { - ConfigHelper.setOutputInitialAddress(conf, initHostAddress); - ConfigHelper.setInputInitialAddress(conf, initHostAddress); - } - - ConfigHelper.setOutputColumnFamily(conf, keyspace, column_family); - CqlConfigHelper.setOutputCql(conf, outputQuery); - - if (bulkOutputFormat) - { - DEFAULT_OUTPUT_FORMAT = BULK_OUTPUT_FORMAT; - if (bulkCfSchema != null) - CqlBulkOutputFormat.setTableSchema(conf, column_family, bulkCfSchema); - else - throw new IOException("bulk_cf_schema is missing in input url parameter"); - if (bulkInsertStatement != null) - CqlBulkOutputFormat.setTableInsertStatement(conf, column_family, bulkInsertStatement); - else - throw new IOException("bulk_insert_statement is missing in input url parameter"); - if (bulkTableAlias != null) - CqlBulkOutputFormat.setTableAlias(conf, bulkTableAlias, column_family); - CqlBulkOutputFormat.setDeleteSourceOnSuccess(conf, bulkDeleteSourceOnSuccess); - if (bulkOutputLocation != null) - conf.set(CqlBulkRecordWriter.OUTPUT_LOCATION, bulkOutputLocation); - if (bulkBuffSize > 0) - conf.set(CqlBulkRecordWriter.BUFFER_SIZE_IN_MB, String.valueOf(bulkBuffSize)); - if (bulkStreamThrottle > 0) - conf.set(CqlBulkRecordWriter.STREAM_THROTTLE_MBITS, String.valueOf(bulkStreamThrottle)); - if (bulkMaxFailedHosts > 0) - conf.set(CqlBulkRecordWriter.MAX_FAILED_HOSTS, String.valueOf(bulkMaxFailedHosts)); - if (partitionerClass!= null) - ConfigHelper.setInputPartitioner(conf, partitionerClass); - } - - setConnectionInformation(); - - if (ConfigHelper.getOutputRpcPort(conf) == 0) - throw new IOException("PIG_OUTPUT_RPC_PORT or PIG_RPC_PORT environment variable not set"); - if (ConfigHelper.getOutputInitialAddress(conf) == null) - throw new IOException("PIG_OUTPUT_INITIAL_ADDRESS or PIG_INITIAL_ADDRESS environment variable not set"); - if (ConfigHelper.getOutputPartitioner(conf) == null) - throw new IOException("PIG_OUTPUT_PARTITIONER or PIG_PARTITIONER environment variable not set"); - - initSchema(storeSignature); - } - - /** Methods to get the column family schema from Cassandra */ - protected void initSchema(String signature) throws IOException - { - Properties properties = UDFContext.getUDFContext().getUDFProperties(CqlNativeStorage.class); - - // Only get the schema if we haven't already gotten it - if (!properties.containsKey(signature)) - { - try (Session client = CqlConfigHelper.getInputCluster(ConfigHelper.getInputInitialAddress(conf), conf).connect()) - { - client.execute("USE " + keyspace); - - // compose the CfDef for the columfamily - TableMetadata cfInfo = getCfInfo(client); - - if (cfInfo != null) - { - properties.setProperty(signature, cfdefToString(cfInfo)); - } - else - throw new IOException(String.format("Table '%s' not found in keyspace '%s'", - column_family, - keyspace)); - } - catch (Exception e) - { - throw new IOException(e); - } - } - } - - - /** convert CfDef to string */ - protected static String cfdefToString(TableMetadata cfDef) throws IOException - { - TableInfo tableInfo = new TableInfo(cfDef); - ByteArrayOutputStream baos = new ByteArrayOutputStream(); - try (ObjectOutputStream oos = new ObjectOutputStream( baos )) - { - oos.writeObject(tableInfo); - } - - return new String( Base64Coder.encode(baos.toByteArray()) ); - } - - /** convert string back to CfDef */ - protected static TableInfo cfdefFromString(String st) throws IOException, ClassNotFoundException - { - byte [] data = Base64Coder.decode( st ); - try (ObjectInputStream ois = new ObjectInputStream(new ByteArrayInputStream(data))) - { - Object o = ois.readObject(); - return (TableInfo)o; - } - } - - /** decompose the query to store the parameters in a map */ - public static Map getQueryMap(String query) throws UnsupportedEncodingException - { - String[] params = query.split("&"); - Map map = new HashMap(params.length); - for (String param : params) - { - String[] keyValue = param.split("="); - map.put(keyValue[0], URLDecoder.decode(keyValue[1], "UTF-8")); - } - return map; - } - - private void setLocationFromUri(String location) throws IOException - { - try - { - if (!location.startsWith("cql://")) - throw new Exception("Bad scheme: " + location); - - String[] urlParts = location.split("\\?"); - if (urlParts.length > 1) - { - Map urlQuery = getQueryMap(urlParts[1]); - - // each page row size - if (urlQuery.containsKey("page_size")) - pageSize = Integer.parseInt(urlQuery.get("page_size")); - - // output prepared statement - if (urlQuery.containsKey("output_query")) - outputQuery = urlQuery.get("output_query"); - - if (urlQuery.containsKey("bulk_output_format")) - bulkOutputFormat = Boolean.valueOf(urlQuery.get("bulk_output_format")); - if (urlQuery.containsKey("bulk_cf_schema")) - bulkCfSchema = urlQuery.get("bulk_cf_schema"); - if (urlQuery.containsKey("bulk_insert_statement")) - bulkInsertStatement = urlQuery.get("bulk_insert_statement"); - if (urlQuery.containsKey("bulk_output_location")) - bulkOutputLocation = urlQuery.get("bulk_output_location"); - if (urlQuery.containsKey("bulk_buff_size")) - bulkBuffSize = Integer.valueOf(urlQuery.get("bulk_buff_size")); - if (urlQuery.containsKey("bulk_stream_throttle")) - bulkStreamThrottle = Integer.valueOf(urlQuery.get("bulk_stream_throttle")); - if (urlQuery.containsKey("bulk_max_failed_hosts")) - bulkMaxFailedHosts = Integer.valueOf(urlQuery.get("bulk_max_failed_hosts")); - if (urlQuery.containsKey("bulk_delete_source")) - bulkDeleteSourceOnSuccess = Boolean.parseBoolean(urlQuery.get("bulk_delete_source")); - if (urlQuery.containsKey("bulk_table_alias")) - bulkTableAlias = urlQuery.get("bulk_table_alias"); - - //split size - if (urlQuery.containsKey("split_size")) - splitSize = Integer.parseInt(urlQuery.get("split_size")); - if (urlQuery.containsKey("partitioner")) - partitionerClass = urlQuery.get("partitioner"); - if (urlQuery.containsKey("use_secondary")) - usePartitionFilter = Boolean.parseBoolean(urlQuery.get("use_secondary")); - if (urlQuery.containsKey("init_address")) - initHostAddress = urlQuery.get("init_address"); - - if (urlQuery.containsKey("native_port")) - nativePort = urlQuery.get("native_port"); - if (urlQuery.containsKey("core_conns")) - nativeCoreConnections = urlQuery.get("core_conns"); - if (urlQuery.containsKey("max_conns")) - nativeMaxConnections = urlQuery.get("max_conns"); - if (urlQuery.containsKey("max_simult_reqs")) - nativeMaxSimultReqs = urlQuery.get("max_simult_reqs"); - if (urlQuery.containsKey("native_timeout")) - nativeConnectionTimeout = urlQuery.get("native_timeout"); - if (urlQuery.containsKey("native_read_timeout")) - nativeReadConnectionTimeout = urlQuery.get("native_read_timeout"); - if (urlQuery.containsKey("rec_buff_size")) - nativeReceiveBufferSize = urlQuery.get("rec_buff_size"); - if (urlQuery.containsKey("send_buff_size")) - nativeSendBufferSize = urlQuery.get("send_buff_size"); - if (urlQuery.containsKey("solinger")) - nativeSolinger = urlQuery.get("solinger"); - if (urlQuery.containsKey("tcp_nodelay")) - nativeTcpNodelay = urlQuery.get("tcp_nodelay"); - if (urlQuery.containsKey("reuse_address")) - nativeReuseAddress = urlQuery.get("reuse_address"); - if (urlQuery.containsKey("keep_alive")) - nativeKeepAlive = urlQuery.get("keep_alive"); - if (urlQuery.containsKey("auth_provider")) - nativeAuthProvider = urlQuery.get("auth_provider"); - if (urlQuery.containsKey("trust_store_path")) - nativeSSLTruststorePath = urlQuery.get("trust_store_path"); - if (urlQuery.containsKey("key_store_path")) - nativeSSLKeystorePath = urlQuery.get("key_store_path"); - if (urlQuery.containsKey("trust_store_password")) - nativeSSLTruststorePassword = urlQuery.get("trust_store_password"); - if (urlQuery.containsKey("key_store_password")) - nativeSSLKeystorePassword = urlQuery.get("key_store_password"); - if (urlQuery.containsKey("cipher_suites")) - nativeSSLCipherSuites = urlQuery.get("cipher_suites"); - if (urlQuery.containsKey("input_cql")) - inputCql = urlQuery.get("input_cql"); - if (urlQuery.containsKey("columns")) - columns = urlQuery.get("columns"); - if (urlQuery.containsKey("where_clause")) - whereClause = urlQuery.get("where_clause"); - if (urlQuery.containsKey("rpc_port")) - rpcPort = urlQuery.get("rpc_port"); - } - String[] parts = urlParts[0].split("/+"); - String[] credentialsAndKeyspace = parts[1].split("@"); - if (credentialsAndKeyspace.length > 1) - { - String[] credentials = credentialsAndKeyspace[0].split(":"); - username = credentials[0]; - password = credentials[1]; - keyspace = credentialsAndKeyspace[1]; - } - else - { - keyspace = parts[1]; - } - column_family = parts[2]; - } - catch (Exception e) - { - throw new IOException("Expected 'cql://[username:password@]/" + - "[?[page_size=][&columns=][&output_query=]" + - "[&where_clause=][&split_size=][&partitioner=][&use_secondary=true|false]" + - "[&init_address=][&native_port=][&core_conns=]" + - "[&max_conns=][&min_simult_reqs=][&max_simult_reqs=]" + - "[&native_timeout=][&native_read_timeout=][&rec_buff_size=]" + - "[&send_buff_size=][&solinger=][&tcp_nodelay=][&reuse_address=]" + - "[&keep_alive=][&auth_provider=][&trust_store_path=]" + - "[&key_store_path=][&trust_store_password=]" + - "[&key_store_password=][&cipher_suites=][&input_cql=]" + - "[columns=][where_clause=]" + - "[&bulk_cf_schema=bulk_cf_schema][&bulk_insert_statement=bulk_insert_statement][&bulk_table_alias=]" + - "[&bulk_output_location=][&bulk_buff_size=][&bulk_delete_source=]" + - "[&bulk_stream_throttle=][&bulk_max_failed_hosts=]]': " + e.getMessage()); - } - } - - public ByteBuffer nullToBB() - { - return ByteBuffer.wrap(new byte[0]); - } - - /** output format */ - public OutputFormat getOutputFormat() throws IOException - { - try - { - return FBUtilities.construct(outputFormatClass, "outputformat"); - } - catch (ConfigurationException e) - { - throw new IOException(e); - } - } - - public void cleanupOnFailure(String failure, Job job) - { - } - - public void cleanupOnSuccess(String location, Job job) throws IOException { - } - - /** return partition keys */ - public String[] getPartitionKeys(String location, Job job) throws IOException - { - if (!usePartitionFilter) - return null; - TableInfo tableMetadata = getCfInfo(loadSignature); - String[] partitionKeys = new String[tableMetadata.getPartitionKey().size()]; - for (int i = 0; i < tableMetadata.getPartitionKey().size(); i++) - { - partitionKeys[i] = tableMetadata.getPartitionKey().get(i).getName(); - } - return partitionKeys; - } - - public void checkSchema(ResourceSchema schema) throws IOException - { - // we don't care about types, they all get casted to ByteBuffers - } - - public ResourceStatistics getStatistics(String location, Job job) - { - return null; - } - - @Override - public InputFormat getInputFormat() throws IOException - { - try - { - return FBUtilities.construct(inputFormatClass, "inputformat"); - } - catch (ConfigurationException e) - { - throw new IOException(e); - } - } - - public String relToAbsPathForStoreLocation(String location, Path curDir) throws IOException - { - return relativeToAbsolutePath(location, curDir); - } - - @Override - public String relativeToAbsolutePath(String location, Path curDir) throws IOException - { - return location; - } - - @Override - public void setUDFContextSignature(String signature) - { - this.loadSignature = signature; - } - - /** StoreFunc methods */ - public void setStoreFuncUDFContextSignature(String signature) - { - this.storeSignature = signature; - } - - /** set hadoop cassandra connection settings */ - protected void setConnectionInformation() throws IOException - { - StorageHelper.setConnectionInformation(conf); - if (System.getenv(StorageHelper.PIG_INPUT_FORMAT) != null) - inputFormatClass = getFullyQualifiedClassName(System.getenv(StorageHelper.PIG_INPUT_FORMAT)); - else - inputFormatClass = DEFAULT_INPUT_FORMAT; - if (System.getenv(StorageHelper.PIG_OUTPUT_FORMAT) != null) - outputFormatClass = getFullyQualifiedClassName(System.getenv(StorageHelper.PIG_OUTPUT_FORMAT)); - else - outputFormatClass = DEFAULT_OUTPUT_FORMAT; - } - - /** get the full class name */ - protected String getFullyQualifiedClassName(String classname) - { - return classname.contains(".") ? classname : "org.apache.cassandra.hadoop." + classname; - } -} - -class TableInfo implements Serializable -{ - private final List columns; - private final List partitionKey; - private final String name; - - public TableInfo(TableMetadata tableMetadata) - { - List cmColumns = tableMetadata.getColumns(); - columns = new ArrayList<>(cmColumns.size()); - for (ColumnMetadata cm : cmColumns) - { - columns.add(new ColumnInfo(this, cm)); - } - List cmPartitionKey = tableMetadata.getPartitionKey(); - partitionKey = new ArrayList<>(cmPartitionKey.size()); - for (ColumnMetadata cm : cmPartitionKey) - { - partitionKey.add(new ColumnInfo(this, cm)); - } - name = tableMetadata.getName(); - } - - public List getPartitionKey() - { - return partitionKey; - } - - public List getColumns() - { - return columns; - } - - public String getName() - { - return name; - } -} - -class ColumnInfo implements Serializable -{ - private final TableInfo table; - private final String name; - private final String typeName; - - public ColumnInfo(TableInfo tableInfo, ColumnMetadata columnMetadata) - { - table = tableInfo; - name = columnMetadata.getName(); - typeName = columnMetadata.getType().toString(); - } - - public String getName() - { - return name; - } - - public String getTypeName() - { - return typeName; - } -} \ No newline at end of file diff --git a/src/java/org/apache/cassandra/hadoop/pig/StorageHelper.java b/src/java/org/apache/cassandra/hadoop/pig/StorageHelper.java deleted file mode 100644 index d700cb7425..0000000000 --- a/src/java/org/apache/cassandra/hadoop/pig/StorageHelper.java +++ /dev/null @@ -1,122 +0,0 @@ -package org.apache.cassandra.hadoop.pig; - -import java.math.BigInteger; -import java.nio.ByteBuffer; -import java.util.Date; -import java.util.UUID; - -import org.apache.cassandra.db.marshal.*; -import org.apache.cassandra.hadoop.ConfigHelper; -import org.apache.cassandra.serializers.CollectionSerializer; -import org.apache.cassandra.utils.ByteBufferUtil; -import org.apache.cassandra.utils.UUIDGen; -import org.apache.hadoop.conf.Configuration; -import org.apache.pig.backend.executionengine.ExecException; -import org.apache.pig.data.DataByteArray; -import org.apache.pig.data.DataType; -import org.apache.pig.data.Tuple; - -@Deprecated -public class StorageHelper -{ - // system environment variables that can be set to configure connection info: - // alternatively, Hadoop JobConf variables can be set using keys from ConfigHelper - public final static String PIG_INPUT_RPC_PORT = "PIG_INPUT_RPC_PORT"; - public final static String PIG_INPUT_INITIAL_ADDRESS = "PIG_INPUT_INITIAL_ADDRESS"; - public final static String PIG_INPUT_PARTITIONER = "PIG_INPUT_PARTITIONER"; - public final static String PIG_OUTPUT_RPC_PORT = "PIG_OUTPUT_RPC_PORT"; - public final static String PIG_OUTPUT_INITIAL_ADDRESS = "PIG_OUTPUT_INITIAL_ADDRESS"; - public final static String PIG_OUTPUT_PARTITIONER = "PIG_OUTPUT_PARTITIONER"; - public final static String PIG_RPC_PORT = "PIG_RPC_PORT"; - public final static String PIG_INITIAL_ADDRESS = "PIG_INITIAL_ADDRESS"; - public final static String PIG_PARTITIONER = "PIG_PARTITIONER"; - public final static String PIG_INPUT_FORMAT = "PIG_INPUT_FORMAT"; - public final static String PIG_OUTPUT_FORMAT = "PIG_OUTPUT_FORMAT"; - public final static String PIG_INPUT_SPLIT_SIZE = "PIG_INPUT_SPLIT_SIZE"; - - - public final static String PARTITION_FILTER_SIGNATURE = "cassandra.partition.filter"; - - protected static void setConnectionInformation(Configuration conf) - { - if (System.getenv(PIG_RPC_PORT) != null) - { - ConfigHelper.setInputRpcPort(conf, System.getenv(PIG_RPC_PORT)); - ConfigHelper.setOutputRpcPort(conf, System.getenv(PIG_RPC_PORT)); - } - - if (System.getenv(PIG_INPUT_RPC_PORT) != null) - ConfigHelper.setInputRpcPort(conf, System.getenv(PIG_INPUT_RPC_PORT)); - if (System.getenv(PIG_OUTPUT_RPC_PORT) != null) - ConfigHelper.setOutputRpcPort(conf, System.getenv(PIG_OUTPUT_RPC_PORT)); - - if (System.getenv(PIG_INITIAL_ADDRESS) != null) - { - ConfigHelper.setInputInitialAddress(conf, System.getenv(PIG_INITIAL_ADDRESS)); - ConfigHelper.setOutputInitialAddress(conf, System.getenv(PIG_INITIAL_ADDRESS)); - } - if (System.getenv(PIG_INPUT_INITIAL_ADDRESS) != null) - ConfigHelper.setInputInitialAddress(conf, System.getenv(PIG_INPUT_INITIAL_ADDRESS)); - if (System.getenv(PIG_OUTPUT_INITIAL_ADDRESS) != null) - ConfigHelper.setOutputInitialAddress(conf, System.getenv(PIG_OUTPUT_INITIAL_ADDRESS)); - - if (System.getenv(PIG_PARTITIONER) != null) - { - ConfigHelper.setInputPartitioner(conf, System.getenv(PIG_PARTITIONER)); - ConfigHelper.setOutputPartitioner(conf, System.getenv(PIG_PARTITIONER)); - } - if(System.getenv(PIG_INPUT_PARTITIONER) != null) - ConfigHelper.setInputPartitioner(conf, System.getenv(PIG_INPUT_PARTITIONER)); - if(System.getenv(PIG_OUTPUT_PARTITIONER) != null) - ConfigHelper.setOutputPartitioner(conf, System.getenv(PIG_OUTPUT_PARTITIONER)); - } - - protected static Object cassandraToObj(AbstractType validator, ByteBuffer value, int nativeProtocolVersion) - { - if (validator instanceof DecimalType || validator instanceof InetAddressType) - return validator.getString(value); - - if (validator instanceof CollectionType) - { - // For CollectionType, the compose() method assumes the v3 protocol format of collection, which - // is not correct here since we query using the CQL-over-thrift interface which use the pre-v3 format - return ((CollectionSerializer)validator.getSerializer()).deserializeForNativeProtocol(value, nativeProtocolVersion); - } - - return validator.compose(value); - } - - /** set the value to the position of the tuple */ - protected static void setTupleValue(Tuple pair, int position, Object value) throws ExecException - { - if (value instanceof BigInteger) - pair.set(position, ((BigInteger) value).intValue()); - else if (value instanceof ByteBuffer) - pair.set(position, new DataByteArray(ByteBufferUtil.getArray((ByteBuffer) value))); - else if (value instanceof UUID) - pair.set(position, new DataByteArray(UUIDGen.decompose((java.util.UUID) value))); - else if (value instanceof Date) - pair.set(position, TimestampType.instance.decompose((Date) value).getLong()); - else - pair.set(position, value); - } - - /** get pig type for the cassandra data type*/ - protected static byte getPigType(AbstractType type) - { - if (type instanceof LongType || type instanceof DateType || type instanceof TimestampType) // DateType is bad and it should feel bad - return DataType.LONG; - else if (type instanceof IntegerType || type instanceof Int32Type) // IntegerType will overflow at 2**31, but is kept for compatibility until pig has a BigInteger - return DataType.INTEGER; - else if (type instanceof AsciiType || type instanceof UTF8Type || type instanceof DecimalType || type instanceof InetAddressType) - return DataType.CHARARRAY; - else if (type instanceof FloatType) - return DataType.FLOAT; - else if (type instanceof DoubleType) - return DataType.DOUBLE; - else if (type instanceof AbstractCompositeType || type instanceof CollectionType) - return DataType.TUPLE; - - return DataType.BYTEARRAY; - } -} diff --git a/test/conf/cassandra_pig.yaml b/test/conf/cassandra_pig.yaml deleted file mode 100644 index ce71410992..0000000000 --- a/test/conf/cassandra_pig.yaml +++ /dev/null @@ -1,42 +0,0 @@ -# -# Warning! -# Consider the effects on 'o.a.c.i.s.LegacySSTableTest' before changing schemas in this file. -# -cluster_name: Test Cluster -memtable_allocation_type: heap_buffers -commitlog_sync: batch -commitlog_sync_batch_window_in_ms: 1.0 -commitlog_segment_size_in_mb: 5 -commitlog_directory: build/test/cassandra/commitlog -hints_directory: build/test/cassandra/hints -partitioner: org.apache.cassandra.dht.Murmur3Partitioner -listen_address: 127.0.0.1 -storage_port: 7010 -rpc_port: 9170 -start_native_transport: true -native_transport_port: 9042 -column_index_size_in_kb: 4 -saved_caches_directory: build/test/cassandra/saved_caches -data_file_directories: - - build/test/cassandra/data -disk_access_mode: mmap -seed_provider: - - class_name: org.apache.cassandra.locator.SimpleSeedProvider - parameters: - - seeds: "127.0.0.1" -endpoint_snitch: org.apache.cassandra.locator.SimpleSnitch -dynamic_snitch: true -request_scheduler: org.apache.cassandra.scheduler.RoundRobinScheduler -request_scheduler_id: keyspace -server_encryption_options: - internode_encryption: none - keystore: conf/.keystore - keystore_password: cassandra - truststore: conf/.truststore - truststore_password: cassandra -incremental_backups: true -concurrent_compactors: 4 -compaction_throughput_mb_per_sec: 0 -row_cache_class_name: org.apache.cassandra.cache.OHCProvider -row_cache_size_in_mb: 16 -enable_user_defined_functions: true diff --git a/test/pig/org/apache/cassandra/pig/CqlRecordReaderTest.java b/test/pig/org/apache/cassandra/pig/CqlRecordReaderTest.java deleted file mode 100644 index b2a74b6a57..0000000000 --- a/test/pig/org/apache/cassandra/pig/CqlRecordReaderTest.java +++ /dev/null @@ -1,150 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.apache.cassandra.pig; - -import java.io.IOException; -import java.util.Iterator; - -import org.junit.Assert; -import org.junit.BeforeClass; -import org.junit.Test; - -import org.apache.cassandra.exceptions.ConfigurationException; -import org.apache.pig.data.Tuple; -import org.apache.thrift.TException; - -public class CqlRecordReaderTest extends PigTestBase -{ - private static String[] statements = { - "DROP KEYSPACE IF EXISTS cql3ks", - "CREATE KEYSPACE cql3ks WITH replication = {'class': 'SimpleStrategy', 'replication_factor' : 1};", - "USE cql3ks;", - - "CREATE TABLE cqltable (" + - "pk1 int," + - "pk2 int," + - "pk3 int," + - "ck1 int," + - "ck2 int," + - "data text," + - "primary key((pk1,pk2,pk3),ck1,ck2));", - "INSERT INTO cqltable(pk1, pk2, pk3, ck1, ck2, data) VALUES (11, 12, 13, 14, 15, 'value1');", - - "CREATE TABLE \"MixedCaseCqlTable\" (" + - "pk1 int," + - "\"PK2\" int," + - "pk3 int," + - "\"CK1\" int," + - "ck2 int," + - "data text," + - "primary key((pk1,\"PK2\",pk3),\"CK1\",ck2));", - "INSERT INTO \"MixedCaseCqlTable\"(pk1, \"PK2\", pk3, \"CK1\", ck2, data) VALUES (11, 12, 13, 14, 15, 'value1');", - }; - - @BeforeClass - public static void setup() throws IOException, ConfigurationException, TException - { - startCassandra(); - executeCQLStatements(statements); - startHadoopCluster(); - } - - @Test - public void defaultCqlQueryTest() throws Exception - { - String initialQuery = "rows = LOAD 'cql://cql3ks/cqltable?" + defaultParameters + nativeParameters + "' USING CqlNativeStorage();"; - pig.registerQuery(initialQuery); - Iterator it = pig.openIterator("rows"); - if (it.hasNext()) { - Tuple t = it.next(); - Assert.assertEquals(t.get(0), 11); - Assert.assertEquals(t.get(1), 12); - Assert.assertEquals(t.get(2), 13); - Assert.assertEquals(t.get(3), 14); - Assert.assertEquals(t.get(4), 15); - Assert.assertEquals(t.get(5), "value1"); - } - else - { - Assert.fail("Failed to get data for query " + initialQuery); - } - } - - @Test - public void defaultMixedCaseCqlQueryTest() throws Exception - { - String initialQuery = "rows = LOAD 'cql://cql3ks/MixedCaseCqlTable?" + defaultParameters + nativeParameters + "' USING CqlNativeStorage();"; - pig.registerQuery(initialQuery); - Iterator it = pig.openIterator("rows"); - if (it.hasNext()) { - Tuple t = it.next(); - Assert.assertEquals(t.get(0), 11); - Assert.assertEquals(t.get(1), 12); - Assert.assertEquals(t.get(2), 13); - Assert.assertEquals(t.get(3), 14); - Assert.assertEquals(t.get(4), 15); - Assert.assertEquals(t.get(5), "value1"); - } - else - { - Assert.fail("Failed to get data for query " + initialQuery); - } - } - - @Test - public void selectColumnsTest() throws Exception - { - String initialQuery = "rows = LOAD 'cql://cql3ks/cqltable?" + defaultParameters + nativeParameters + "&columns=ck1%2Cck2%2Cdata' USING CqlNativeStorage();"; - pig.registerQuery(initialQuery); - Iterator it = pig.openIterator("rows"); - if (it.hasNext()) { - Tuple t = it.next(); - Assert.assertEquals(t.get(0), 11); - Assert.assertEquals(t.get(1), 12); - Assert.assertEquals(t.get(2), 13); - Assert.assertEquals(t.get(3), 14); - Assert.assertEquals(t.get(4), 15); - Assert.assertEquals(t.get(5), "value1"); - } - else - { - Assert.fail("Failed to get data for query " + initialQuery); - } - } - - @Test - public void whereClauseTest() throws Exception - { - String initialQuery = "rows = LOAD 'cql://cql3ks/cqltable?" + defaultParameters + nativeParameters + "&where_clause=ck1%3d14' USING CqlNativeStorage();"; - pig.registerQuery(initialQuery); - Iterator it = pig.openIterator("rows"); - if (it.hasNext()) { - Tuple t = it.next(); - Assert.assertEquals(t.get(0), 11); - Assert.assertEquals(t.get(1), 12); - Assert.assertEquals(t.get(2), 13); - Assert.assertEquals(t.get(3), 14); - Assert.assertEquals(t.get(4), 15); - Assert.assertEquals(t.get(5), "value1"); - } - else - { - Assert.fail("Failed to get data for query " + initialQuery); - } - } -} diff --git a/test/pig/org/apache/cassandra/pig/CqlTableDataTypeTest.java b/test/pig/org/apache/cassandra/pig/CqlTableDataTypeTest.java deleted file mode 100644 index ca01901747..0000000000 --- a/test/pig/org/apache/cassandra/pig/CqlTableDataTypeTest.java +++ /dev/null @@ -1,479 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.cassandra.pig; - -import java.io.IOException; -import java.util.Iterator; - -import org.apache.cassandra.db.marshal.TimeUUIDType; -import org.apache.cassandra.db.marshal.UUIDType; -import org.apache.cassandra.exceptions.ConfigurationException; -import org.apache.cassandra.utils.Hex; -import org.apache.pig.data.DataByteArray; -import org.apache.pig.data.Tuple; -import org.apache.thrift.TException; - -import org.junit.Assert; -import org.junit.BeforeClass; -import org.junit.Test; - -public class CqlTableDataTypeTest extends PigTestBase -{ - //ASCII (AsciiType.instance), - //BIGINT (LongType.instance), - //BLOB (BytesType.instance), - //BOOLEAN (BooleanType.instance), - //COUNTER (CounterColumnType.instance), - //DECIMAL (DecimalType.instance), - //DOUBLE (DoubleType.instance), - //FLOAT (FloatType.instance), - //INET (InetAddressType.instance), - //INT (Int32Type.instance), - //TEXT (UTF8Type.instance), - //TIMESTAMP(DateType.instance), - //UUID (UUIDType.instance), - //VARCHAR (UTF8Type.instance), - //VARINT (IntegerType.instance), - //TIMEUUID (TimeUUIDType.instance); - //SET - //LIST - //MAP - //Create table to test the above data types - private static String[] statements = { - "DROP KEYSPACE IF EXISTS cql3ks", - "CREATE KEYSPACE cql3ks WITH replication = {'class': 'SimpleStrategy', 'replication_factor' : 1}", - "USE cql3ks;", - - "CREATE TABLE cqltable (" + - "key int primary key," + - "col_ascii ascii," + - "col_bigint bigint," + - "col_blob blob," + - "col_boolean boolean," + - "col_decimal decimal," + - "col_double double," + - "col_float float," + - "col_inet inet," + - "col_int int," + - "col_text text," + - "col_timestamp timestamp," + - "col_uuid uuid," + - "col_varchar varchar," + - "col_varint varint," + - "col_timeuuid timeuuid);", - - "CREATE TABLE settable (" + - "key int primary key," + - "col_set_ascii set," + - "col_set_bigint set," + - "col_set_blob set," + - "col_set_boolean set," + - "col_set_decimal set," + - "col_set_double set," + - "col_set_float set," + - "col_set_inet set," + - "col_set_int set," + - "col_set_text set," + - "col_set_timestamp set," + - "col_set_uuid set," + - "col_set_varchar set," + - "col_set_varint set," + - "col_set_timeuuid set);", - - "CREATE TABLE listtable (" + - "key int primary key," + - "col_list_ascii list," + - "col_list_bigint list," + - "col_list_blob list," + - "col_list_boolean list," + - "col_list_decimal list," + - "col_list_double list," + - "col_list_float list," + - "col_list_inet list," + - "col_list_int list," + - "col_list_text list," + - "col_list_timestamp list," + - "col_list_uuid list," + - "col_list_varchar list," + - "col_list_varint list," + - "col_list_timeuuid list);", - - "CREATE TABLE maptable (" + - "key int primary key," + - "col_map_ascii map," + - "col_map_bigint map," + - "col_map_blob map," + - "col_map_boolean map," + - "col_map_decimal map," + - "col_map_double map," + - "col_map_float map," + - "col_map_inet map," + - "col_map_int map," + - "col_map_text map," + - "col_map_timestamp map," + - "col_map_uuid map," + - "col_map_varchar map," + - "col_map_varint map," + - "col_map_timeuuid map);", - - "INSERT INTO cqltable(key, col_ascii) VALUES (1, 'ascii');", - "INSERT INTO cqltable(key, col_bigint) VALUES (1, 12345678);", - "INSERT INTO cqltable(key, col_blob) VALUES (1, 0x23446c6c6f);", - "INSERT INTO cqltable(key, col_boolean) VALUES (1, false);", - "INSERT INTO cqltable(key, col_decimal) VALUES (1, 23.4567);", - "INSERT INTO cqltable(key, col_double) VALUES (1, 12345678.12345678);", - "INSERT INTO cqltable(key, col_float) VALUES (1, 123.12);", - "INSERT INTO cqltable(key, col_inet) VALUES (1, '127.0.0.1');", - "INSERT INTO cqltable(key, col_int) VALUES (1, 123);", - "INSERT INTO cqltable(key, col_text) VALUES (1, 'text');", - "INSERT INTO cqltable(key, col_timestamp) VALUES (1, '2011-02-03T04:05:00+0000');", - "INSERT INTO cqltable(key, col_timeuuid) VALUES (1, maxTimeuuid('2013-01-01 00:05+0000'));", - "INSERT INTO cqltable(key, col_uuid) VALUES (1, 550e8400-e29b-41d4-a716-446655440000);", - "INSERT INTO cqltable(key, col_varchar) VALUES (1, 'varchar');", - "INSERT INTO cqltable(key, col_varint) VALUES (1, 123);", - - "INSERT INTO settable(key, col_set_ascii) VALUES (1, {'ascii1', 'ascii2'});", - "INSERT INTO settable(key, col_set_bigint) VALUES (1, {12345678, 12345679});", - "INSERT INTO settable(key, col_set_blob) VALUES (1, {0x68656c6c6f, 0x68656c6c6e});", - "INSERT INTO settable(key, col_set_boolean) VALUES (1, {false, true});", - "INSERT INTO settable(key, col_set_decimal) VALUES (1, {23.4567, 23.4568});", - "INSERT INTO settable(key, col_set_double) VALUES (1, {12345678.12345678, 12345678.12345679});", - "INSERT INTO settable(key, col_set_float) VALUES (1, {123.12, 123.13});", - "INSERT INTO settable(key, col_set_inet) VALUES (1, {'127.0.0.1', '127.0.0.2'});", - "INSERT INTO settable(key, col_set_int) VALUES (1, {123, 124});", - "INSERT INTO settable(key, col_set_text) VALUES (1, {'text1', 'text2'});", - "INSERT INTO settable(key, col_set_timestamp) VALUES (1, {'2011-02-03T04:05:00+0000', '2011-02-04T04:05:00+0000'});", - "INSERT INTO settable(key, col_set_timeuuid) VALUES (1, {e23f450f-53a6-11e2-7f7f-7f7f7f7f7f7f, e23f450f-53a6-11e2-7f7f-7f7f7f7f7f77});", - "INSERT INTO settable(key, col_set_uuid) VALUES (1, {550e8400-e29b-41d4-a716-446655440000, 550e8400-e29b-41d4-a716-446655440001});", - "INSERT INTO settable(key, col_set_varchar) VALUES (1, {'varchar1', 'varchar2'});", - "INSERT INTO settable(key, col_set_varint) VALUES (1, {123, 124});", - - "INSERT INTO listtable(key, col_list_ascii) VALUES (1, ['ascii2', 'ascii1']);", - "INSERT INTO listtable(key, col_list_bigint) VALUES (1, [12345679, 12345678]);", - "INSERT INTO listtable(key, col_list_blob) VALUES (1, [0x68656c6c6e, 0x68656c6c6f]);", - "INSERT INTO listtable(key, col_list_boolean) VALUES (1, [true, false]);", - "INSERT INTO listtable(key, col_list_decimal) VALUES (1, [23.4568, 23.4567]);", - "INSERT INTO listtable(key, col_list_double) VALUES (1, [12345678.12345679, 12345678.12345678]);", - "INSERT INTO listtable(key, col_list_float) VALUES (1, [123.13, 123.12]);", - "INSERT INTO listtable(key, col_list_inet) VALUES (1, ['127.0.0.2', '127.0.0.1']);", - "INSERT INTO listtable(key, col_list_int) VALUES (1, [124, 123]);", - "INSERT INTO listtable(key, col_list_text) VALUES (1, ['text2', 'text1']);", - "INSERT INTO listtable(key, col_list_timestamp) VALUES (1, ['2011-02-04T04:05:00+0000', '2011-02-03T04:05:00+0000']);", - "INSERT INTO listtable(key, col_list_timeuuid) VALUES (1, [e23f450f-53a6-11e2-7f7f-7f7f7f7f7f77, e23f450f-53a6-11e2-7f7f-7f7f7f7f7f7f]);", - "INSERT INTO listtable(key, col_list_uuid) VALUES (1, [550e8400-e29b-41d4-a716-446655440001, 550e8400-e29b-41d4-a716-446655440000]);", - "INSERT INTO listtable(key, col_list_varchar) VALUES (1, ['varchar2', 'varchar1']);", - "INSERT INTO listtable(key, col_list_varint) VALUES (1, [124, 123]);", - - "INSERT INTO maptable(key, col_map_ascii) VALUES (1, {'ascii1' : 'ascii2'});", - "INSERT INTO maptable(key, col_map_bigint) VALUES (1, {12345678 : 12345679});", - "INSERT INTO maptable(key, col_map_blob) VALUES (1, {0x68656c6c6f : 0x68656c6c6e});", - "INSERT INTO maptable(key, col_map_boolean) VALUES (1, {false : true});", - "INSERT INTO maptable(key, col_map_decimal) VALUES (1, {23.4567 : 23.4568});", - "INSERT INTO maptable(key, col_map_double) VALUES (1, {12345678.12345678 : 12345678.12345679});", - "INSERT INTO maptable(key, col_map_float) VALUES (1, {123.12 : 123.13});", - "INSERT INTO maptable(key, col_map_inet) VALUES (1, {'127.0.0.1' : '127.0.0.2'});", - "INSERT INTO maptable(key, col_map_int) VALUES (1, {123 : 124});", - "INSERT INTO maptable(key, col_map_text) VALUES (1, {'text1' : 'text2'});", - "INSERT INTO maptable(key, col_map_timestamp) VALUES (1, {'2011-02-03T04:05:00+0000' : '2011-02-04T04:05:00+0000'});", - "INSERT INTO maptable(key, col_map_timeuuid) VALUES (1, {e23f450f-53a6-11e2-7f7f-7f7f7f7f7f7f : e23f450f-53a6-11e2-7f7f-7f7f7f7f7f77});", - "INSERT INTO maptable(key, col_map_uuid) VALUES (1, {550e8400-e29b-41d4-a716-446655440000 : 550e8400-e29b-41d4-a716-446655440001});", - "INSERT INTO maptable(key, col_map_varchar) VALUES (1, {'varchar1' : 'varchar2'});", - "INSERT INTO maptable(key, col_map_varint) VALUES (1, {123 : 124});", - - "CREATE TABLE countertable (key int primary key, col_counter counter);", - "UPDATE countertable SET col_counter = col_counter + 3 WHERE key = 1;", - }; - - @BeforeClass - public static void setup() throws IOException, ConfigurationException, TException - { - startCassandra(); - executeCQLStatements(statements); - startHadoopCluster(); - } - - @Test - public void testCqlNativeStorageRegularType() throws IOException - { - //input_cql=select * from cqltable where token(key) > ? and token(key) <= ? - cqlTableTest("rows = LOAD 'cql://cql3ks/cqltable?" + defaultParameters + nativeParameters + "&input_cql=select%20*%20from%20cqltable%20where%20token(key)%20%3E%20%3F%20and%20token(key)%20%3C%3D%20%3F' USING CqlNativeStorage();"); - - //input_cql=select * from countertable where token(key) > ? and token(key) <= ? - counterTableTest("cc_rows = LOAD 'cql://cql3ks/countertable?" + defaultParameters + nativeParameters + "&input_cql=select%20*%20from%20countertable%20where%20token(key)%20%3E%20%3F%20and%20token(key)%20%3C%3D%20%3F' USING CqlNativeStorage();"); - } - - private void cqlTableTest(String initialQuery) throws IOException - { - pig.registerQuery(initialQuery); - Iterator it = pig.openIterator("rows"); - //{key: int, - //col_ascii: chararray, - //col_bigint: long, - //col_blob: bytearray, - //col_boolean: bytearray, - //col_decimal: chararray, - //col_double: double, - //col_float: float, - //col_inet: chararray, - //col_int: int, - //col_text: chararray, - //col_timestamp: long, - //col_timeuuid: bytearray, - //col_uuid: chararray, - //col_varchar: chararray, - //col_varint: int} - if (it.hasNext()) { - Tuple t = it.next(); - Assert.assertEquals(t.get(0), 1); - Assert.assertEquals(t.get(1), "ascii"); - Assert.assertEquals(t.get(2), 12345678L); - Assert.assertEquals(t.get(3), new DataByteArray(Hex.hexToBytes("23446c6c6f"))); - Assert.assertEquals(t.get(4), false); - Assert.assertEquals(t.get(5), "23.4567"); - Assert.assertEquals(t.get(6), 12345678.12345678d); - Assert.assertEquals(t.get(7), 123.12f); - Assert.assertEquals(t.get(8), "127.0.0.1"); - Assert.assertEquals(t.get(9), 123); - Assert.assertEquals(t.get(10), "text"); - Assert.assertEquals(t.get(11), 1296705900000L); - Assert.assertEquals(t.get(12), new DataByteArray((TimeUUIDType.instance.fromString("e23f450f-53a6-11e2-7f7f-7f7f7f7f7f7f").array()))); - Assert.assertEquals(t.get(13), new DataByteArray((UUIDType.instance.fromString("550e8400-e29b-41d4-a716-446655440000").array()))); - Assert.assertEquals(t.get(14), "varchar"); - Assert.assertEquals(t.get(15), 123); - } - else - { - Assert.fail("Failed to get data for query " + initialQuery); - } - } - - private void counterTableTest(String initialQuery) throws IOException - { - pig.registerQuery(initialQuery); - Iterator it = pig.openIterator("cc_rows"); - if (it.hasNext()) { - Tuple t = it.next(); - Assert.assertEquals(t.get(0), 1); - Assert.assertEquals(t.get(1), 3L); - } - else - { - Assert.fail("Failed to get data for query " + initialQuery); - } - } - - @Test - public void testCqlNativeStorageSetType() throws IOException - { - //input_cql=select * from settable where token(key) > ? and token(key) <= ? - settableTest("set_rows = LOAD 'cql://cql3ks/settable?" + defaultParameters + nativeParameters + "&input_cql=select%20*%20from%20settable%20where%20token(key)%20%3E%20%3F%20and%20token(key)%20%3C%3D%20%3F' USING CqlNativeStorage();"); - } - - private void settableTest(String initialQuery) throws IOException - { - pig.registerQuery(initialQuery); - Iterator it = pig.openIterator("set_rows"); - if (it.hasNext()) { - Tuple t = it.next(); - Assert.assertEquals(t.get(0), 1); - Tuple innerTuple = (Tuple) t.get(1); - Assert.assertEquals(innerTuple.get(0), "ascii1"); - Assert.assertEquals(innerTuple.get(1), "ascii2"); - innerTuple = (Tuple) t.get(2); - Assert.assertEquals(innerTuple.get(0), 12345678L); - Assert.assertEquals(innerTuple.get(1), 12345679L); - innerTuple = (Tuple) t.get(3); - Assert.assertEquals(innerTuple.get(0), new DataByteArray(Hex.hexToBytes("68656c6c6e"))); - Assert.assertEquals(innerTuple.get(1), new DataByteArray(Hex.hexToBytes("68656c6c6f"))); - innerTuple = (Tuple) t.get(4); - Assert.assertEquals(innerTuple.get(0), false); - Assert.assertEquals(innerTuple.get(1), true); - innerTuple = (Tuple) t.get(5); - Assert.assertEquals(innerTuple.get(0), "23.4567"); - Assert.assertEquals(innerTuple.get(1), "23.4568"); - innerTuple = (Tuple) t.get(6); - Assert.assertEquals(innerTuple.get(0), 12345678.12345678d); - Assert.assertEquals(innerTuple.get(1), 12345678.12345679d); - innerTuple = (Tuple) t.get(7); - Assert.assertEquals(innerTuple.get(0), 123.12f); - Assert.assertEquals(innerTuple.get(1), 123.13f); - innerTuple = (Tuple) t.get(8); - Assert.assertEquals(innerTuple.get(0), "127.0.0.1"); - Assert.assertEquals(innerTuple.get(1), "127.0.0.2"); - innerTuple = (Tuple) t.get(9); - Assert.assertEquals(innerTuple.get(0), 123); - Assert.assertEquals(innerTuple.get(1), 124); - innerTuple = (Tuple) t.get(10); - Assert.assertEquals(innerTuple.get(0), "text1"); - Assert.assertEquals(innerTuple.get(1), "text2"); - innerTuple = (Tuple) t.get(11); - Assert.assertEquals(innerTuple.get(0), 1296705900000L); - Assert.assertEquals(innerTuple.get(1), 1296792300000L); - innerTuple = (Tuple) t.get(12); - Assert.assertEquals(innerTuple.get(0), new DataByteArray((TimeUUIDType.instance.fromString("e23f450f-53a6-11e2-7f7f-7f7f7f7f7f77").array()))); - Assert.assertEquals(innerTuple.get(1), new DataByteArray((TimeUUIDType.instance.fromString("e23f450f-53a6-11e2-7f7f-7f7f7f7f7f7f").array()))); - innerTuple = (Tuple) t.get(13); - Assert.assertEquals(innerTuple.get(0), new DataByteArray((UUIDType.instance.fromString("550e8400-e29b-41d4-a716-446655440000").array()))); - Assert.assertEquals(innerTuple.get(1), new DataByteArray((UUIDType.instance.fromString("550e8400-e29b-41d4-a716-446655440001").array()))); - innerTuple = (Tuple) t.get(14); - Assert.assertEquals(innerTuple.get(0), "varchar1"); - Assert.assertEquals(innerTuple.get(1), "varchar2"); - innerTuple = (Tuple) t.get(15); - Assert.assertEquals(innerTuple.get(0), 123); - Assert.assertEquals(innerTuple.get(1), 124); - } - else - { - Assert.fail("Failed to get data for query " + initialQuery); - } - } - - @Test - public void testCqlNativeStorageListType() throws IOException - { - //input_cql=select * from listtable where token(key) > ? and token(key) <= ? - listtableTest("list_rows = LOAD 'cql://cql3ks/listtable?" + defaultParameters + nativeParameters + "&input_cql=select%20*%20from%20listtable%20where%20token(key)%20%3E%20%3F%20and%20token(key)%20%3C%3D%20%3F' USING CqlNativeStorage();"); - } - - private void listtableTest(String initialQuery) throws IOException - { - pig.registerQuery(initialQuery); - Iterator it = pig.openIterator("list_rows"); - if (it.hasNext()) { - Tuple t = it.next(); - Assert.assertEquals(t.get(0), 1); - Tuple innerTuple = (Tuple) t.get(1); - Assert.assertEquals(innerTuple.get(1), "ascii1"); - Assert.assertEquals(innerTuple.get(0), "ascii2"); - innerTuple = (Tuple) t.get(2); - Assert.assertEquals(innerTuple.get(1), 12345678L); - Assert.assertEquals(innerTuple.get(0), 12345679L); - innerTuple = (Tuple) t.get(3); - Assert.assertEquals(innerTuple.get(1), new DataByteArray(Hex.hexToBytes("68656c6c6f"))); - Assert.assertEquals(innerTuple.get(0), new DataByteArray(Hex.hexToBytes("68656c6c6e"))); - innerTuple = (Tuple) t.get(4); - Assert.assertEquals(innerTuple.get(1), false); - Assert.assertEquals(innerTuple.get(0), true); - innerTuple = (Tuple) t.get(5); - Assert.assertEquals(innerTuple.get(1), "23.4567"); - Assert.assertEquals(innerTuple.get(0), "23.4568"); - innerTuple = (Tuple) t.get(6); - Assert.assertEquals(innerTuple.get(1), 12345678.12345678d); - Assert.assertEquals(innerTuple.get(0), 12345678.12345679d); - innerTuple = (Tuple) t.get(7); - Assert.assertEquals(innerTuple.get(1), 123.12f); - Assert.assertEquals(innerTuple.get(0), 123.13f); - innerTuple = (Tuple) t.get(8); - Assert.assertEquals(innerTuple.get(1), "127.0.0.1"); - Assert.assertEquals(innerTuple.get(0), "127.0.0.2"); - innerTuple = (Tuple) t.get(9); - Assert.assertEquals(innerTuple.get(1), 123); - Assert.assertEquals(innerTuple.get(0), 124); - innerTuple = (Tuple) t.get(10); - Assert.assertEquals(innerTuple.get(1), "text1"); - Assert.assertEquals(innerTuple.get(0), "text2"); - innerTuple = (Tuple) t.get(11); - Assert.assertEquals(innerTuple.get(1), 1296705900000L); - Assert.assertEquals(innerTuple.get(0), 1296792300000L); - innerTuple = (Tuple) t.get(12); - Assert.assertEquals(innerTuple.get(1), new DataByteArray((TimeUUIDType.instance.fromString("e23f450f-53a6-11e2-7f7f-7f7f7f7f7f7f").array()))); - Assert.assertEquals(innerTuple.get(0), new DataByteArray((TimeUUIDType.instance.fromString("e23f450f-53a6-11e2-7f7f-7f7f7f7f7f77").array()))); - innerTuple = (Tuple) t.get(13); - Assert.assertEquals(innerTuple.get(1), new DataByteArray((UUIDType.instance.fromString("550e8400-e29b-41d4-a716-446655440000").array()))); - Assert.assertEquals(innerTuple.get(0), new DataByteArray((UUIDType.instance.fromString("550e8400-e29b-41d4-a716-446655440001").array()))); - innerTuple = (Tuple) t.get(14); - Assert.assertEquals(innerTuple.get(1), "varchar1"); - Assert.assertEquals(innerTuple.get(0), "varchar2"); - innerTuple = (Tuple) t.get(15); - Assert.assertEquals(innerTuple.get(1), 123); - Assert.assertEquals(innerTuple.get(0), 124); - } - else - { - Assert.fail("Failed to get data for query " + initialQuery); - } - } - - @Test - public void testCqlNativeStorageMapType() throws IOException - { - //input_cql=select * from maptable where token(key) > ? and token(key) <= ? - maptableTest("map_rows = LOAD 'cql://cql3ks/maptable?" + defaultParameters + nativeParameters + "&input_cql=select%20*%20from%20maptable%20where%20token(key)%20%3E%20%3F%20and%20token(key)%20%3C%3D%20%3F' USING CqlNativeStorage();"); - } - - private void maptableTest(String initialQuery) throws IOException - { - pig.registerQuery(initialQuery); - Iterator it = pig.openIterator("map_rows"); - if (it.hasNext()) { - Tuple t = it.next(); - Assert.assertEquals(t.get(0), 1); - Tuple innerTuple = (Tuple) ((Tuple) t.get(1)).get(0); - Assert.assertEquals(innerTuple.get(0), "ascii1"); - Assert.assertEquals(innerTuple.get(1), "ascii2"); - innerTuple = (Tuple) ((Tuple) t.get(2)).get(0); - Assert.assertEquals(innerTuple.get(0), 12345678L); - Assert.assertEquals(innerTuple.get(1), 12345679L); - innerTuple = (Tuple) ((Tuple) t.get(3)).get(0); - Assert.assertEquals(innerTuple.get(0), new DataByteArray(Hex.hexToBytes("68656c6c6f"))); - Assert.assertEquals(innerTuple.get(1), new DataByteArray(Hex.hexToBytes("68656c6c6e"))); - innerTuple = (Tuple) ((Tuple) t.get(4)).get(0); - Assert.assertEquals(innerTuple.get(0), false); - Assert.assertEquals(innerTuple.get(1), true); - innerTuple = (Tuple) ((Tuple) t.get(5)).get(0); - Assert.assertEquals(innerTuple.get(0), "23.4567"); - Assert.assertEquals(innerTuple.get(1), "23.4568"); - innerTuple = (Tuple) ((Tuple) t.get(6)).get(0); - Assert.assertEquals(innerTuple.get(0), 12345678.12345678d); - Assert.assertEquals(innerTuple.get(1), 12345678.12345679d); - innerTuple = (Tuple) ((Tuple) t.get(7)).get(0); - Assert.assertEquals(innerTuple.get(0), 123.12f); - Assert.assertEquals(innerTuple.get(1), 123.13f); - innerTuple = (Tuple) ((Tuple) t.get(8)).get(0); - Assert.assertEquals(innerTuple.get(0), "127.0.0.1"); - Assert.assertEquals(innerTuple.get(1), "127.0.0.2"); - innerTuple = (Tuple) ((Tuple) t.get(9)).get(0); - Assert.assertEquals(innerTuple.get(0), 123); - Assert.assertEquals(innerTuple.get(1), 124); - innerTuple = (Tuple) ((Tuple) t.get(10)).get(0); - Assert.assertEquals(innerTuple.get(0), "text1"); - Assert.assertEquals(innerTuple.get(1), "text2"); - innerTuple = (Tuple) ((Tuple) t.get(11)).get(0); - Assert.assertEquals(innerTuple.get(0), 1296705900000L); - Assert.assertEquals(innerTuple.get(1), 1296792300000L); - innerTuple = (Tuple) ((Tuple) t.get(12)).get(0); - Assert.assertEquals(innerTuple.get(0), new DataByteArray((TimeUUIDType.instance.fromString("e23f450f-53a6-11e2-7f7f-7f7f7f7f7f7f").array()))); - Assert.assertEquals(innerTuple.get(1), new DataByteArray((TimeUUIDType.instance.fromString("e23f450f-53a6-11e2-7f7f-7f7f7f7f7f77").array()))); - innerTuple = (Tuple) ((Tuple) t.get(13)).get(0); - Assert.assertEquals(innerTuple.get(0), new DataByteArray((UUIDType.instance.fromString("550e8400-e29b-41d4-a716-446655440000").array()))); - Assert.assertEquals(innerTuple.get(1), new DataByteArray((UUIDType.instance.fromString("550e8400-e29b-41d4-a716-446655440001").array()))); - innerTuple = (Tuple) ((Tuple) t.get(14)).get(0); - Assert.assertEquals(innerTuple.get(0), "varchar1"); - Assert.assertEquals(innerTuple.get(1), "varchar2"); - innerTuple = (Tuple) ((Tuple) t.get(15)).get(0); - Assert.assertEquals(innerTuple.get(0), 123); - Assert.assertEquals(innerTuple.get(1), 124); - } - else - { - Assert.fail("Failed to get data for query " + initialQuery); - } - } - -} diff --git a/test/pig/org/apache/cassandra/pig/CqlTableTest.java b/test/pig/org/apache/cassandra/pig/CqlTableTest.java deleted file mode 100644 index 3902fce7d9..0000000000 --- a/test/pig/org/apache/cassandra/pig/CqlTableTest.java +++ /dev/null @@ -1,303 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - */ -package org.apache.cassandra.pig; - -import java.io.IOException; -import java.util.Iterator; - -import org.apache.cassandra.exceptions.ConfigurationException; -import org.apache.pig.data.Tuple; -import org.apache.thrift.TException; -import org.junit.Assert; -import org.junit.BeforeClass; -import org.junit.Test; - -public class CqlTableTest extends PigTestBase -{ - private static String[] statements = { - "DROP KEYSPACE IF EXISTS cql3ks", - "CREATE KEYSPACE cql3ks WITH replication = {'class': 'SimpleStrategy', 'replication_factor' : 1}", - "USE cql3ks;", - - "CREATE TABLE cqltable (key1 text, key2 int, column1 int, column2 float, primary key(key1, key2))", - "INSERT INTO cqltable (key1, key2, column1, column2) values ('key1', 111, 100, 10.1)", - "CREATE TABLE compactcqltable (key1 text, column1 int, column2 float, primary key(key1)) WITH COMPACT STORAGE", - "INSERT INTO compactcqltable (key1, column1, column2) values ('key1', 100, 10.1)", - - "CREATE TABLE test (a int PRIMARY KEY, b int);", - "CREATE INDEX test_b on test (b);", - - "CREATE TABLE moredata (x int PRIMARY KEY, y int);", - "CREATE TABLE test_bulk (a int PRIMARY KEY, b int);", - "INSERT INTO test_bulk (a,b) VALUES (1,1);", - "INSERT INTO test_bulk (a,b) VALUES (2,2);", - "INSERT INTO test_bulk (a,b) VALUES (3,3);", - "INSERT INTO test (a,b) VALUES (1,1);", - "INSERT INTO test (a,b) VALUES (2,2);", - "INSERT INTO test (a,b) VALUES (3,3);", - "INSERT INTO moredata (x, y) VALUES (4,4);", - "INSERT INTO moredata (x, y) VALUES (5,5);", - "INSERT INTO moredata (x, y) VALUES (6,6);", - - "CREATE TABLE compotable (a int, b int, c text, d text, PRIMARY KEY (a,b,c));", - "INSERT INTO compotable (a, b , c , d ) VALUES ( 1,1,'One','match');", - "INSERT INTO compotable (a, b , c , d ) VALUES ( 2,2,'Two','match');", - "INSERT INTO compotable (a, b , c , d ) VALUES ( 3,3,'Three','match');", - "INSERT INTO compotable (a, b , c , d ) VALUES ( 4,4,'Four','match');", - - "create table compmore (id int PRIMARY KEY, x int, y int, z text, data text);", - "INSERT INTO compmore (id, x, y, z,data) VALUES (1,5,6,'Fix','nomatch');", - "INSERT INTO compmore (id, x, y, z,data) VALUES (2,6,5,'Sive','nomatch');", - "INSERT INTO compmore (id, x, y, z,data) VALUES (3,7,7,'Seven','match');", - "INSERT INTO compmore (id, x, y, z,data) VALUES (4,8,8,'Eight','match');", - "INSERT INTO compmore (id, x, y, z,data) VALUES (5,9,10,'Ninen','nomatch');", - - "CREATE TABLE collectiontable(m text PRIMARY KEY, n map);", - "UPDATE collectiontable SET n['key1'] = 'value1' WHERE m = 'book1';", - "UPDATE collectiontable SET n['key2'] = 'value2' WHERE m = 'book2';", - "UPDATE collectiontable SET n['key3'] = 'value3' WHERE m = 'book3';", - "UPDATE collectiontable SET n['key4'] = 'value4' WHERE m = 'book4';", - "CREATE TABLE nulltable(m text PRIMARY KEY, n map);", - "UPDATE nulltable SET n['key1'] = 'value1' WHERE m = 'book1';", - "UPDATE nulltable SET n['key2'] = 'value2' WHERE m = 'book2';", - "UPDATE nulltable SET n['key3'] = 'value3' WHERE m = 'book3';", - "UPDATE nulltable SET n['key4'] = 'value4' WHERE m = 'book4';", - }; - - @BeforeClass - public static void setup() throws IOException, ConfigurationException, TException - { - startCassandra(); - executeCQLStatements(statements); - startHadoopCluster(); - } - - @Test - public void testCqlNativeStorageSchema() throws IOException - { - //input_cql=select * from cqltable where token(key1) > ? and token(key1) <= ? - cqlTableSchemaTest("rows = LOAD 'cql://cql3ks/cqltable?" + defaultParameters + nativeParameters + "&input_cql=select%20*%20from%20cqltable%20where%20token(key1)%20%3E%20%3F%20and%20token(key1)%20%3C%3D%20%3F' USING CqlNativeStorage();"); - - //input_cql=select * from compactcqltable where token(key1) > ? and token(key1) <= ? - compactCqlTableSchemaTest("rows = LOAD 'cql://cql3ks/compactcqltable?" + defaultParameters + nativeParameters + "&input_cql=select%20*%20from%20compactcqltable%20where%20token(key1)%20%3E%20%3F%20and%20token(key1)%20%3C%3D%20%3F' USING CqlNativeStorage();"); - } - - private void compactCqlTableSchemaTest(String initialQuery) throws IOException - { - pig.registerQuery(initialQuery); - Iterator it = pig.openIterator("rows"); - if (it.hasNext()) { - Tuple t = it.next(); - Assert.assertEquals(t.get(0).toString(), "key1"); - Assert.assertEquals(t.get(1), 100); - Assert.assertEquals(t.get(2), 10.1f); - Assert.assertEquals(3, t.size()); - } - else - { - Assert.fail("Failed to get data for query " + initialQuery); - } - } - - private void cqlTableSchemaTest(String initialQuery) throws IOException - { - pig.registerQuery(initialQuery); - Iterator it = pig.openIterator("rows"); - if (it.hasNext()) { - Tuple t = it.next(); - Assert.assertEquals(t.get(0).toString(), "key1"); - Assert.assertEquals(t.get(1), 111); - Assert.assertEquals(t.get(2), 100); - Assert.assertEquals(t.get(3), 10.1f); - Assert.assertEquals(4, t.size()); - } - else - { - Assert.fail("Failed to get data for query " + initialQuery); - } - } - - @Test - public void testCqlNativeStorageSingleKeyTable() throws IOException - { - //input_cql=select * from moredata where token(x) > ? and token(x) <= ? - singleKeyTableTest("moretestvalues= LOAD 'cql://cql3ks/moredata?" + defaultParameters + nativeParameters + "&input_cql=select%20*%20from%20moredata%20where%20token(x)%20%3E%20%3F%20and%20token(x)%20%3C%3D%20%3F' USING CqlNativeStorage();"); - } - - private void singleKeyTableTest(String initialQuery) throws IOException - { - pig.setBatchOn(); - pig.registerQuery(initialQuery); - pig.registerQuery("insertformat= FOREACH moretestvalues GENERATE TOTUPLE(TOTUPLE('a',x)),TOTUPLE(y);"); - pig.registerQuery("STORE insertformat INTO 'cql://cql3ks/test?" + defaultParameters + nativeParameters + "&output_query=UPDATE+cql3ks.test+set+b+%3D+%3F' USING CqlNativeStorage();"); - pig.executeBatch(); - //(5,5) - //(6,6) - //(4,4) - //(2,2) - //(3,3) - //(1,1) - //input_cql=select * from test where token(a) > ? and token(a) <= ? - pig.registerQuery("result= LOAD 'cql://cql3ks/test?" + defaultParameters + nativeParameters + "&input_cql=select%20*%20from%20test%20where%20token(a)%20%3E%20%3F%20and%20token(a)%20%3C%3D%20%3F' USING CqlNativeStorage();"); - Iterator it = pig.openIterator("result"); - int count = 0; - while (it.hasNext()) { - Tuple t = it.next(); - Assert.assertEquals(t.get(0), t.get(1)); - count ++; - } - Assert.assertEquals(6, count); - } - - @Test - public void testCqlNativeStorageCompositeKeyTable() throws IOException - { - //input_cql=select * from compmore where token(id) > ? and token(id) <= ? - compositeKeyTableTest("moredata= LOAD 'cql://cql3ks/compmore?" + defaultParameters + nativeParameters + "&input_cql=select%20*%20from%20compmore%20where%20token(id)%20%3E%20%3F%20and%20token(id)%20%3C%3D%20%3F' USING CqlNativeStorage();"); - } - - private void compositeKeyTableTest(String initialQuery) throws IOException - { - pig.setBatchOn(); - pig.registerQuery(initialQuery); - pig.registerQuery("insertformat = FOREACH moredata GENERATE TOTUPLE (TOTUPLE('a',x),TOTUPLE('b',y), TOTUPLE('c',z)),TOTUPLE(data);"); - pig.registerQuery("STORE insertformat INTO 'cql://cql3ks/compotable?" + defaultParameters + nativeParameters + "&output_query=UPDATE%20cql3ks.compotable%20SET%20d%20%3D%20%3F' USING CqlNativeStorage();"); - pig.executeBatch(); - - //(5,6,Fix,nomatch) - //(3,3,Three,match) - //(1,1,One,match) - //(2,2,Two,match) - //(7,7,Seven,match) - //(8,8,Eight,match) - //(6,5,Sive,nomatch) - //(4,4,Four,match) - //(9,10,Ninen,nomatch) - //input_cql=select * from compotable where token(a) > ? and token(a) <= ? - pig.registerQuery("result= LOAD 'cql://cql3ks/compotable?" + defaultParameters + nativeParameters + "&input_cql=select%20*%20from%20compotable%20where%20token(a)%20%3E%20%3F%20and%20token(a)%20%3C%3D%20%3F' USING CqlNativeStorage();"); - Iterator it = pig.openIterator("result"); - int count = 0; - while (it.hasNext()) { - it.next(); - count ++; - } - Assert.assertEquals(count, 9); - } - - @Test - public void testCqlNativeStorageCollectionColumnTable() throws IOException - { - //input_cql=select * from collectiontable where token(m) > ? and token(m) <= ? - CollectionColumnTableTest("collectiontable= LOAD 'cql://cql3ks/collectiontable?" + defaultParameters + nativeParameters + "&input_cql=select%20*%20from%20collectiontable%20where%20token(m)%20%3E%20%3F%20and%20token(m)%20%3C%3D%20%3F' USING CqlNativeStorage();"); - } - - private void CollectionColumnTableTest(String initialQuery) throws IOException - { - pig.setBatchOn(); - pig.registerQuery(initialQuery); - pig.registerQuery("recs= FOREACH collectiontable GENERATE TOTUPLE(TOTUPLE('m', m) ), TOTUPLE(TOTUPLE('map', TOTUPLE('m', 'mm'), TOTUPLE('n', 'nn')));"); - pig.registerQuery("STORE recs INTO 'cql://cql3ks/collectiontable?" + defaultParameters + nativeParameters + "&output_query=update+cql3ks.collectiontable+set+n+%3D+%3F' USING CqlNativeStorage();"); - pig.executeBatch(); - - //(book2,((m,mm),(n,nn))) - //(book3,((m,mm),(n,nn))) - //(book4,((m,mm),(n,nn))) - //(book1,((m,mm),(n,nn))) - //input_cql=select * from collectiontable where token(m) > ? and token(m) <= ? - pig.registerQuery("result= LOAD 'cql://cql3ks/collectiontable?" + defaultParameters + nativeParameters + "&input_cql=select%20*%20from%20collectiontable%20where%20token(m)%20%3E%20%3F%20and%20token(m)%20%3C%3D%20%3F' USING CqlNativeStorage();"); - Iterator it = pig.openIterator("result"); - if (it.hasNext()) { - Tuple t = it.next(); - Tuple t1 = (Tuple) t.get(1); - Assert.assertEquals(t1.size(), 2); - Tuple element1 = (Tuple) t1.get(0); - Tuple element2 = (Tuple) t1.get(1); - Assert.assertEquals(element1.get(0), "m"); - Assert.assertEquals(element1.get(1), "mm"); - Assert.assertEquals(element2.get(0), "n"); - Assert.assertEquals(element2.get(1), "nn"); - } - else - { - Assert.fail("Can't fetch any data"); - } - } - - @Test - public void testCqlNativeStorageNullTuples() throws IOException - { - //input_cql=select * from collectiontable where token(m) > ? and token(m) <= ? - NullTupleTest("nulltable= LOAD 'cql://cql3ks/collectiontable?" + defaultParameters + nativeParameters + "&input_cql=select%20*%20from%20nulltable%20where%20token(m)%20%3E%20%3F%20and%20token(m)%20%3C%3D%20%3F' USING CqlNativeStorage();"); - } - - private void NullTupleTest(String initialQuery) throws IOException - { - pig.setBatchOn(); - pig.registerQuery(initialQuery); - pig.registerQuery("recs= FOREACH nulltable GENERATE TOTUPLE(TOTUPLE('m', m) ), TOTUPLE(TOTUPLE('map', TOTUPLE('m', null), TOTUPLE('n', null)));"); - pig.registerQuery("STORE recs INTO 'cql://cql3ks/nulltable?" + defaultParameters + nativeParameters + "&output_query=update+cql3ks.nulltable+set+n+%3D+%3F' USING CqlNativeStorage();"); - pig.executeBatch(); - - pig.registerQuery("result= LOAD 'cql://cql3ks/nulltable?" + defaultParameters + nativeParameters + "&input_cql=select%20*%20from%20nulltable%20where%20token(m)%20%3E%20%3F%20and%20token(m)%20%3C%3D%20%3F' USING CqlNativeStorage();"); - Iterator it = pig.openIterator("result"); - if (it.hasNext()) { - Tuple t = it.next(); - Tuple t1 = (Tuple) t.get(1); - Assert.assertEquals(t1.size(), 2); - Tuple element1 = (Tuple) t1.get(0); - Tuple element2 = (Tuple) t1.get(1); - Assert.assertEquals(element1.get(0), "m"); - Assert.assertEquals(element1.get(1), ""); - Assert.assertEquals(element2.get(0), "n"); - Assert.assertEquals(element2.get(1), ""); - } - else - { - Assert.fail("Can't fetch any data"); - } - } - - @Test - public void testCqlStorageSingleKeyTableBulkLoad() throws TException, IOException - { - pig.setBatchOn(); - //input_cql=select * from moredata where token(x) > ? and token(x) <= ? - pig.registerQuery("moretestvalues= LOAD 'cql://cql3ks/moredata?" + defaultParameters + nativeParameters + "&input_cql=select%20*%20from%20moredata%20where%20token(x)%20%3E%20%3F%20and%20token(x)%20%3C%3D%20%3F' USING CqlNativeStorage();"); - pig.registerQuery("insertformat= FOREACH moretestvalues GENERATE TOTUPLE(x, y);"); - pig.registerQuery("STORE insertformat INTO 'cql://cql3ks/test_bulk?" + defaultParameters + nativeParameters + "&bulk_output_format=true&bulk_cf_schema=CREATE%20TABLE%20cql3ks.test_bulk%20(a%20int%20PRIMARY%20KEY%2C%20b%20int)&bulk_insert_statement=Insert%20into%20cql3ks.test_bulk(a%2C%20b)%20values(%3F%2C%3F)' USING CqlNativeStorage();"); - pig.executeBatch(); - - //(5,5) - //(6,6) - //(4,4) - //(2,2) - //(3,3) - //(1,1) - //input_cql=select * from test_bulk1 where token(a) > ? and token(a) <= ? - pig.registerQuery("result= LOAD 'cql://cql3ks/test_bulk?" + defaultParameters + nativeParameters + "&input_cql=select%20*%20from%20test_bulk%20where%20token(a)%20%3E%20%3F%20and%20token(a)%20%3C%3D%20%3F' USING CqlNativeStorage();"); - Iterator it = pig.openIterator("result"); - int count = 0; - while (it.hasNext()) { - Tuple t = it.next(); - Assert.assertEquals(t.get(0), t.get(1)); - count ++; - } - Assert.assertEquals(6, count); - } -} diff --git a/test/pig/org/apache/cassandra/pig/PigTestBase.java b/test/pig/org/apache/cassandra/pig/PigTestBase.java deleted file mode 100644 index 067397bd6b..0000000000 --- a/test/pig/org/apache/cassandra/pig/PigTestBase.java +++ /dev/null @@ -1,117 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - */ -package org.apache.cassandra.pig; - -import java.io.IOException; - -import com.datastax.driver.core.Cluster; -import com.datastax.driver.core.Session; -import org.apache.cassandra.SchemaLoader; -import org.apache.cassandra.config.Schema; -import org.apache.cassandra.db.marshal.AbstractType; -import org.apache.cassandra.db.marshal.TypeParser; -import org.apache.cassandra.exceptions.ConfigurationException; -import org.apache.cassandra.exceptions.SyntaxException; -import org.apache.cassandra.service.EmbeddedCassandraService; -import org.apache.cassandra.utils.ByteBufferUtil; -import org.apache.hadoop.conf.Configuration; -import org.apache.pig.ExecType; -import org.apache.pig.PigServer; -import org.apache.pig.backend.hadoop.datastorage.ConfigurationUtil; -import org.apache.pig.impl.PigContext; -import org.apache.pig.test.MiniCluster; -import org.junit.After; -import org.junit.AfterClass; -import org.junit.Before; - -public class PigTestBase extends SchemaLoader -{ - protected static EmbeddedCassandraService cassandra; - protected static Configuration conf; - protected static MiniCluster cluster; - protected static PigServer pig; - protected static String defaultParameters= "init_address=localhost&rpc_port=9170&partitioner=org.apache.cassandra.dht.Murmur3Partitioner"; - protected static String nativeParameters = "&core_conns=2&max_conns=10&min_simult_reqs=3&max_simult_reqs=10&native_timeout=10000000" + - "&native_read_timeout=10000000&send_buff_size=4096&receive_buff_size=4096&solinger=3" + - "&tcp_nodelay=true&reuse_address=true&keep_alive=true&native_port=9042"; - - static - { - System.setProperty("logback.configurationFile", "logback-test.xml"); - System.setProperty("cassandra.config", "cassandra_pig.yaml"); - } - - @AfterClass - public static void oneTimeTearDown() throws Exception { - cluster.shutDown(); - } - - @Before - public void beforeTest() throws Exception { - pig = new PigServer(new PigContext(ExecType.LOCAL, ConfigurationUtil.toProperties(conf))); - PigContext.initializeImportList("org.apache.cassandra.hadoop.pig"); - } - - @After - public void tearDown() throws Exception { - pig.shutdown(); - } - - protected static Session getClient() - { - Cluster cluster = Cluster.builder().addContactPoints("localhost").withPort(9042).build(); - return cluster.connect(); - } - - protected static void startCassandra() throws IOException - { - Schema.instance.clear(); // Schema are now written on disk and will be reloaded - cassandra = new EmbeddedCassandraService(); - cassandra.start(); - } - - protected static void startHadoopCluster() - { - cluster = MiniCluster.buildCluster(); - conf = cluster.getConfiguration(); - } - - protected AbstractType parseType(String type) throws IOException - { - try - { - return TypeParser.parse(type); - } - catch (ConfigurationException | SyntaxException e) - { - throw new IOException(e); - } - } - - protected static void executeCQLStatements(String[] statements) - { - Session client = getClient(); - - for (String statement : statements) - { - System.out.println("Executing statement: " + statement); - client.execute(statement); - } - } -} diff --git a/test/pig/org/apache/cassandra/pig/ThriftColumnFamilyDataTypeTest.java b/test/pig/org/apache/cassandra/pig/ThriftColumnFamilyDataTypeTest.java deleted file mode 100644 index 273cdffbc2..0000000000 --- a/test/pig/org/apache/cassandra/pig/ThriftColumnFamilyDataTypeTest.java +++ /dev/null @@ -1,150 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - */ -package org.apache.cassandra.pig; - -import java.io.IOException; - -import org.apache.cassandra.db.marshal.TimeUUIDType; -import org.apache.cassandra.db.marshal.UUIDType; -import org.apache.cassandra.exceptions.ConfigurationException; -import org.apache.cassandra.utils.Hex; -import org.apache.pig.data.DataByteArray; -import org.apache.pig.data.Tuple; -import org.junit.BeforeClass; -import org.junit.Test; - -import static junit.framework.Assert.assertEquals; - -public class ThriftColumnFamilyDataTypeTest extends PigTestBase -{ - private static String[] statements = { - "DROP KEYSPACE IF EXISTS thrift_ks", - "CREATE KEYSPACE thrift_ks WITH replication = {'class': 'SimpleStrategy', 'replication_factor': 1};", - "USE thrift_ks;", - - "CREATE TABLE some_app (" + - "key text PRIMARY KEY," + - "col_ascii ascii," + - "col_bigint bigint," + - "col_blob blob," + - "col_boolean boolean," + - "col_decimal decimal," + - "col_double double," + - "col_float float," + - "col_inet inet," + - "col_int int," + - "col_text text," + - "col_timestamp timestamp," + - "col_timeuuid timeuuid," + - "col_uuid uuid," + - "col_varint varint)" + - " WITH COMPACT STORAGE;", - - "INSERT INTO some_app (key, col_ascii, col_bigint, col_blob, col_boolean, col_decimal, col_double, col_float," + - "col_inet, col_int, col_text, col_timestamp, col_uuid, col_varint, col_timeuuid) " + - "VALUES ('foo', 'ascii', 12345678, 0xDEADBEEF, false, 23.345, 2.7182818284590451, 23.45, '127.0.0.1', 23, 'hello', " + - "'2011-02-03T04:05:00+0000', 550e8400-e29b-41d4-a716-446655440000, 12345, e23f450f-53a6-11e2-7f7f-7f7f7f7f7f7f);", - - "CREATE TABLE cc (key text, name text, value counter, PRIMARY KEY (key, name)) WITH COMPACT STORAGE", - - "UPDATE cc SET value = value + 3 WHERE key = 'chuck' AND name = 'kick'", - }; - - @BeforeClass - public static void setup() throws IOException, ConfigurationException - { - startCassandra(); - executeCQLStatements(statements); - startHadoopCluster(); - } - - @Test - public void testCassandraStorageDataType() throws IOException - { - pig.registerQuery("rows = LOAD 'cql://thrift_ks/some_app?" + defaultParameters + "' USING CqlNativeStorage();"); - Tuple t = pig.openIterator("rows").next(); - - // key - assertEquals("foo", t.get(0)); - - // col_ascii - Object column = t.get(1); - assertEquals("ascii", column); - - // col_bigint - column = t.get(2); - assertEquals(12345678L, column); - - // col_blob - column = t.get(3); - assertEquals(new DataByteArray(Hex.hexToBytes("DEADBEEF")), column); - - // col_boolean - column = t.get(4); - assertEquals(false, column); - - // col_decimal - column = t.get(5); - assertEquals("23.345", column); - - // col_double - column = t.get(6); - assertEquals(2.7182818284590451d, column); - - // col_float - column = t.get(7); - assertEquals(23.45f, column); - - // col_inet - column = t.get(8); - assertEquals("127.0.0.1", column); - - // col_int - column = t.get(9); - assertEquals(23, column); - - // col_text - column = t.get(10); - assertEquals("hello", column); - - // col_timestamp - column = t.get(11); - assertEquals(1296705900000L, column); - - // col_timeuuid - column = t.get(12); - assertEquals(new DataByteArray((TimeUUIDType.instance.fromString("e23f450f-53a6-11e2-7f7f-7f7f7f7f7f7f").array())), column); - - // col_uuid - column = t.get(13); - assertEquals(new DataByteArray((UUIDType.instance.fromString("550e8400-e29b-41d4-a716-446655440000").array())), column); - - // col_varint - column = t.get(14); - assertEquals(12345, column); - - pig.registerQuery("cc_rows = LOAD 'cql://thrift_ks/cc?" + defaultParameters + "' USING CqlNativeStorage();"); - t = pig.openIterator("cc_rows").next(); - - assertEquals("chuck", t.get(0)); - - assertEquals("kick", t.get(1)); - assertEquals(3L, t.get(2)); - } -} diff --git a/test/pig/org/apache/cassandra/pig/ThriftColumnFamilyTest.java b/test/pig/org/apache/cassandra/pig/ThriftColumnFamilyTest.java deleted file mode 100644 index 93ac5be354..0000000000 --- a/test/pig/org/apache/cassandra/pig/ThriftColumnFamilyTest.java +++ /dev/null @@ -1,617 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - */ -package org.apache.cassandra.pig; - -import java.io.IOException; -import java.nio.ByteBuffer; -import java.util.Iterator; - -import com.datastax.driver.core.ResultSet; -import com.datastax.driver.core.Row; -import com.datastax.driver.core.Session; -import org.apache.cassandra.exceptions.ConfigurationException; -import org.apache.cassandra.utils.ByteBufferUtil; -import org.apache.pig.data.DataBag; -import org.apache.pig.data.DataByteArray; -import org.apache.pig.data.Tuple; - -import org.junit.Assert; -import org.junit.BeforeClass; -import org.junit.Test; - -public class ThriftColumnFamilyTest extends PigTestBase -{ - private static String[] statements = { - "DROP KEYSPACE IF EXISTS thrift_ks", - "CREATE KEYSPACE thrift_ks WITH replication = {'class': 'SimpleStrategy', 'replication_factor': 1};", - "USE thrift_ks;", - - "CREATE TABLE some_app (" + - "key text PRIMARY KEY," + - "name text," + - "vote_type text," + - "rating int," + - "score bigint," + - "percent float," + - "atomic_weight double," + - "created timestamp)" + - " WITH COMPACT STORAGE;", - - "CREATE INDEX ON some_app(name);", - - "INSERT INTO some_app (key, name, vote_type, rating, score, percent, atomic_weight, created) " + - "VALUES ('foo', 'User Foo', 'like', 8, 125000, 85.0, 2.7182818284590451, 1335890877);", - - "INSERT INTO some_app (key, name, vote_type, rating, score, percent, atomic_weight, created) " + - "VALUES ('bar', 'User Bar', 'like', 9, 15000, 35.0, 3.1415926535897931, 1335890877);", - - "INSERT INTO some_app (key, name, vote_type, rating, score, percent, atomic_weight, created) " + - "VALUES ('baz', 'User Baz', 'dislike', 3, 512000, 95.3, 1.61803399, 1335890877);", - - "INSERT INTO some_app (key, name, vote_type, rating, score, percent, atomic_weight, created) " + - "VALUES ('qux', 'User Qux', 'dislike', 2, 12000, 64.7, 0.660161815846869, 1335890877);", - - "CREATE TABLE copy_of_some_app (" + - "key text PRIMARY KEY," + - "name text," + - "vote_type text," + - "rating int," + - "score bigint," + - "percent float," + - "atomic_weight double," + - "created timestamp)" + - " WITH COMPACT STORAGE;", - - "CREATE INDEX ON copy_of_some_app(name);", - - "CREATE TABLE u8 (" + - "key text," + - "column1 text," + - "value blob," + - "PRIMARY KEY (key, column1))" + - " WITH COMPACT STORAGE", - - "INSERT INTO u8 (key, column1, value) VALUES ('foo', 'x', asciiAsBlob('Z'))", - - "CREATE TABLE bytes (" + - "key blob," + - "column1 text," + - "value blob," + - "PRIMARY KEY (key, column1))" + - " WITH COMPACT STORAGE", - - "INSERT INTO bytes (key, column1, value) VALUES (asciiAsBlob('foo'), 'x', asciiAsBlob('Z'))", - - "CREATE TABLE cc (key text, name text, value counter, PRIMARY KEY (key, name)) WITH COMPACT STORAGE", - - "UPDATE cc SET value = value + 3 WHERE key = 'chuck' AND name = 'kick'", - "UPDATE cc SET value = value + 1 WHERE key = 'chuck' AND name = 'fist'", - - "CREATE TABLE compo (" + - "key text," + - "column1 text," + - "column2 text," + - "value text," + - "PRIMARY KEY (key, column1, column2))" + - " WITH COMPACT STORAGE", - - "INSERT INTO compo (key, column1, column2, value) VALUES ('punch', 'bruce', 'lee', 'ouch');", - "INSERT INTO compo (key, column1, column2, value) VALUES ('punch', 'bruce', 'bruce', 'hunh?');", - "INSERT INTO compo (key, column1, column2, value) VALUES ('kick', 'bruce', 'lee', 'oww');", - "INSERT INTO compo (key, column1, column2, value) VALUES ('kick', 'bruce', 'bruce', 'watch it, mate');", - - "CREATE TABLE compo_int (" + - "key text," + - "column1 bigint," + - "column2 bigint," + - "value text," + - "PRIMARY KEY (key, column1, column2))" + - " WITH COMPACT STORAGE", - - "INSERT INTO compo_int (key, column1, column2, value) VALUES ('clock', 1, 0, 'z');", - "INSERT INTO compo_int (key, column1, column2, value) VALUES ('clock', 1, 30, 'zzzz');", - "INSERT INTO compo_int (key, column1, column2, value) VALUES ('clock', 2, 30, 'daddy?');", - "INSERT INTO compo_int (key, column1, column2, value) VALUES ('clock', 6, 30, 'coffee...');", - - "CREATE TABLE compo_int_copy (" + - "key text," + - "column1 bigint," + - "column2 bigint," + - "value text," + - "PRIMARY KEY (key, column1, column2))" + - " WITH COMPACT STORAGE", - - "CREATE TABLE compo_key (" + - "key text," + - "column1 bigint," + - "column2 bigint," + - "value text," + - "PRIMARY KEY ((key, column1), column2))" + - " WITH COMPACT STORAGE", - - "INSERT INTO compo_key (key, column1, column2, value) VALUES ('clock', 10, 1, 'z');", - "INSERT INTO compo_key (key, column1, column2, value) VALUES ('clock', 20, 1, 'zzzz');", - "INSERT INTO compo_key (key, column1, column2, value) VALUES ('clock', 30, 2, 'daddy?');", - "INSERT INTO compo_key (key, column1, column2, value) VALUES ('clock', 40, 6, 'coffee...');", - - "CREATE TABLE compo_key_copy (" + - "key text," + - "column1 bigint," + - "column2 bigint," + - "value text," + - "PRIMARY KEY ((key, column1), column2))" + - " WITH COMPACT STORAGE", - }; - - private static String[] deleteCopyOfSomeAppTableData = { - "use thrift_ks;", - "DELETE FROM copy_of_some_app WHERE key = 'foo';", - "DELETE FROM copy_of_some_app WHERE key = 'bar';", - "DELETE FROM copy_of_some_app WHERE key = 'baz';", - "DELETE FROM copy_of_some_app WHERE key = 'qux';", - }; - - @BeforeClass - public static void setup() throws IOException, ConfigurationException - { - startCassandra(); - executeCQLStatements(statements); - startHadoopCluster(); - } - - @Test - public void testCqlNativeStorage() throws IOException - { - //regular thrift column families - //input_cql=select * from "some_app" where token(key) > ? and token(key) <= ? - cqlStorageTest("data = load 'cql://thrift_ks/some_app?" + defaultParameters + nativeParameters + "&input_cql=select%20*%20from%20%22some_app%22%20where%20token(key)%20%3E%20%3F%20and%20token(key)%20%3C%3D%20%3F' using CqlNativeStorage();"); - - //Test counter column family - //input_cql=select * from "cc" where token(key) > ? and token(key) <= ? - cqlStorageCounterTableTest("cc_data = load 'cql://thrift_ks/cc?" + defaultParameters + nativeParameters + "&input_cql=select%20*%20from%20%22cc%22%20where%20token(key)%20%3E%20%3F%20and%20token(key)%20%3C%3D%20%3F' using CqlNativeStorage();"); - - //Test composite column family - //input_cql=select * from "compo" where token(key) > ? and token(key) <= ? - cqlStorageCompositeTableTest("compo_data = load 'cql://thrift_ks/compo?" + defaultParameters + nativeParameters + "&input_cql=select%20*%20from%20%22compo%22%20where%20token(key)%20%3E%20%3F%20and%20token(key)%20%3C%3D%20%3F' using CqlNativeStorage();"); - } - - private void cqlStorageTest(String initialQuery) throws IOException - { - pig.registerQuery(initialQuery); - - //(bar,3.141592653589793,1335890877,User Bar,35.0,9,15000,like) - //(baz,1.61803399,1335890877,User Baz,95.3,3,512000,dislike) - //(foo,2.718281828459045,1335890877,User Foo,85.0,8,125000,like) - //(qux,0.660161815846869,1335890877,User Qux,64.7,2,12000,dislike) - - //{key: chararray,atomic_weight: double,created: long,name: chararray,percent: float,rating: int,score: long,vote_type: chararray} - Iterator it = pig.openIterator("data"); - int count = 0; - while (it.hasNext()) { - count ++; - Tuple t = it.next(); - if ("bar".equals(t.get(0))) - { - Assert.assertEquals(t.get(1), 3.141592653589793d); - Assert.assertEquals(t.get(3), "User Bar"); - Assert.assertEquals(t.get(4), 35.0f); - Assert.assertEquals(t.get(5), 9); - Assert.assertEquals(t.get(6), 15000L); - Assert.assertEquals(t.get(7), "like"); - } - else if ("baz".equals(t.get(0))) - { - Assert.assertEquals(t.get(1), 1.61803399d); - Assert.assertEquals(t.get(3), "User Baz"); - Assert.assertEquals(t.get(4), 95.3f); - Assert.assertEquals(t.get(5), 3); - Assert.assertEquals(t.get(6), 512000L); - Assert.assertEquals(t.get(7), "dislike"); - } - else if ("foo".equals(t.get(0))) - { - Assert.assertEquals(t.get(0), "foo"); - Assert.assertEquals(t.get(1), 2.718281828459045d); - Assert.assertEquals(t.get(3), "User Foo"); - Assert.assertEquals(t.get(4), 85.0f); - Assert.assertEquals(t.get(5), 8); - Assert.assertEquals(t.get(6), 125000L); - Assert.assertEquals(t.get(7), "like"); - } - else if ("qux".equals(t.get(0))) - { - Assert.assertEquals(t.get(0), "qux"); - Assert.assertEquals(t.get(1), 0.660161815846869d); - Assert.assertEquals(t.get(3), "User Qux"); - Assert.assertEquals(t.get(4), 64.7f); - Assert.assertEquals(t.get(5), 2); - Assert.assertEquals(t.get(6), 12000L); - Assert.assertEquals(t.get(7), "dislike"); - } - } - Assert.assertEquals(count, 4); - } - - private void cqlStorageCounterTableTest(String initialQuery) throws IOException - { - pig.registerQuery(initialQuery); - - //(chuck,fist,1) - //(chuck,kick,3) - - // {key: chararray,column1: chararray,value: long} - Iterator it = pig.openIterator("cc_data"); - int count = 0; - while (it.hasNext()) { - count ++; - Tuple t = it.next(); - if ("chuck".equals(t.get(0)) && "fist".equals(t.get(1))) - Assert.assertEquals(t.get(2), 1L); - else if ("chuck".equals(t.get(0)) && "kick".equals(t.get(1))) - Assert.assertEquals(t.get(2), 3L); - } - Assert.assertEquals(count, 2); - } - - private void cqlStorageCompositeTableTest(String initialQuery) throws IOException - { - pig.registerQuery(initialQuery); - - //(kick,bruce,bruce,watch it, mate) - //(kick,bruce,lee,oww) - //(punch,bruce,bruce,hunh?) - //(punch,bruce,lee,ouch) - - //{key: chararray,column1: chararray,column2: chararray,value: chararray} - Iterator it = pig.openIterator("compo_data"); - int count = 0; - while (it.hasNext()) { - count ++; - Tuple t = it.next(); - if ("kick".equals(t.get(0)) && "bruce".equals(t.get(1)) && "bruce".equals(t.get(2))) - Assert.assertEquals(t.get(3), "watch it, mate"); - else if ("kick".equals(t.get(0)) && "bruce".equals(t.get(1)) && "lee".equals(t.get(2))) - Assert.assertEquals(t.get(3), "oww"); - else if ("punch".equals(t.get(0)) && "bruce".equals(t.get(1)) && "bruce".equals(t.get(2))) - Assert.assertEquals(t.get(3), "hunh?"); - else if ("punch".equals(t.get(0)) && "bruce".equals(t.get(1)) && "lee".equals(t.get(2))) - Assert.assertEquals(t.get(3), "ouch"); - } - Assert.assertEquals(count, 4); - } - - @Test - public void testCqlNativeStorageSchema() throws IOException - { - //results: (qux,(atomic_weight,0.660161815846869),(created,1335890877),(name,User Qux),(percent,64.7), - //(rating,2),(score,12000),(vote_type,dislike)) - pig.registerQuery("rows = LOAD 'cql://thrift_ks/some_app?" + defaultParameters + "' USING CqlNativeStorage();"); - - //schema: {key: chararray,atomic_weight: (name: chararray,value: double),created: (name: chararray,value: long), - //name: (name: chararray,value: chararray),percent: (name: chararray,value: float), - //rating: (name: chararray,value: int),score: (name: chararray,value: long), - //vote_type: (name: chararray,value: chararray),columns: {(name: chararray,value: chararray)}} - Iterator it = pig.openIterator("rows"); - if (it.hasNext()) { - Tuple t = it.next(); - String rowKey = t.get(0).toString(); - if ("qux".equals(rowKey)) - { - Tuple column = (Tuple) t.get(1); - Assert.assertEquals(column.get(0), "atomic_weight"); - Assert.assertEquals(column.get(1), 0.660161815846869d); - column = (Tuple) t.get(3); - Assert.assertEquals(column.get(0), "name"); - Assert.assertEquals(column.get(1), "User Qux"); - column = (Tuple) t.get(4); - Assert.assertEquals(column.get(0), "percent"); - Assert.assertEquals(column.get(1), 64.7f); - column = (Tuple) t.get(5); - Assert.assertEquals(column.get(0), "rating"); - Assert.assertEquals(column.get(1), 2); - column = (Tuple) t.get(6); - Assert.assertEquals(column.get(0), "score"); - Assert.assertEquals(column.get(1), 12000L); - column = (Tuple) t.get(7); - Assert.assertEquals(column.get(0), "vote_type"); - Assert.assertEquals(column.get(1), "dislike"); - } - } - } - - @Test - public void testCqlNativeStorageFullCopy() throws IOException - { - pig.setBatchOn(); - pig.registerQuery("rows = LOAD 'cql://thrift_ks/some_app?" + defaultParameters + nativeParameters + "&input_cql=select%20*%20from%20some_app%20where%20token(key)%20%3E%20%3F%20and%20token(key)%20%3C%3D%20%3F' USING CqlNativeStorage();"); - pig.registerQuery("records = FOREACH rows GENERATE TOTUPLE(TOTUPLE('key', key)),TOTUPLE(atomic_weight, created, name, percent, rating, score, vote_type);"); - //full copy - pig.registerQuery("STORE records INTO 'cql://thrift_ks/copy_of_some_app?" + defaultParameters + nativeParameters + "&output_query=UPDATE+thrift_ks.copy_of_some_app+set+atomic_weight+%3D+%3F,+created+%3D+%3F,+name+%3D+%3F,+percent+%3D+%3F,+rating+%3D+%3F,+score+%3D+%3F,+vote_type+%3D+%3F' USING CqlNativeStorage();"); - pig.executeBatch(); - Assert.assertEquals("User Qux", getColumnValue("thrift_ks", "copy_of_some_app", "name", "qux", "UTF8Type")); - Assert.assertEquals("dislike", getColumnValue("thrift_ks", "copy_of_some_app", "vote_type", "qux", "UTF8Type")); - Assert.assertEquals("64.7", getColumnValue("thrift_ks", "copy_of_some_app", "percent", "qux", "FloatType")); - } - - @Test - public void testCqlNativeStorageSingleTupleCopy() throws IOException - { - executeCQLStatements(deleteCopyOfSomeAppTableData); - pig.setBatchOn(); - pig.registerQuery("rows = LOAD 'cql://thrift_ks/some_app?" + defaultParameters + nativeParameters + "&input_cql=select%20*%20from%20some_app%20where%20token(key)%20%3E%20%3F%20and%20token(key)%20%3C%3D%20%3F' USING CqlNativeStorage();"); - //single tuple - pig.registerQuery("onecol = FOREACH rows GENERATE TOTUPLE(TOTUPLE('key', key)), TOTUPLE(percent);"); - pig.registerQuery("STORE onecol INTO 'cql://thrift_ks/copy_of_some_app?" + defaultParameters + nativeParameters + "&output_query=UPDATE+thrift_ks.copy_of_some_app+set+percent+%3D+%3F' USING CqlNativeStorage();"); - pig.executeBatch(); - String value = getColumnValue("thrift_ks", "copy_of_some_app", "name", "qux", "UTF8Type"); - if (value != null) - Assert.fail(); - value = getColumnValue("thrift_ks", "copy_of_some_app", "vote_type", "qux", "UTF8Type"); - if (value != null) - Assert.fail(); - Assert.assertEquals("64.7", getColumnValue("thrift_ks", "copy_of_some_app", "percent", "qux", "FloatType")); - } - - @Test - public void testCqlNativeStorageBagOnlyCopy() throws IOException - { - executeCQLStatements(deleteCopyOfSomeAppTableData); - pig.setBatchOn(); - pig.registerQuery("rows = LOAD 'cql://thrift_ks/some_app?" + defaultParameters + nativeParameters + "&input_cql=select%20*%20from%20some_app%20where%20token(key)%20%3E%20%3F%20and%20token(key)%20%3C%3D%20%3F' USING CqlNativeStorage();"); - //bag only - pig.registerQuery("other = FOREACH rows GENERATE TOTUPLE(TOTUPLE('key', key)), TOTUPLE();"); - pig.registerQuery("STORE other INTO 'cql://thrift_ks/copy_of_some_app?" + defaultParameters + nativeParameters + "' USING CqlNativeStorage();"); - pig.executeBatch(); - String value = getColumnValue("thrift_ks", "copy_of_some_app", "name", "qux", "UTF8Type"); - if (value != null) - Assert.fail(); - value = getColumnValue("thrift_ks", "copy_of_some_app", "vote_type", "qux", "UTF8Type"); - if (value != null) - Assert.fail(); - value = getColumnValue("thrift_ks", "copy_of_some_app", "percent", "qux", "FloatType"); - if (value != null) - Assert.fail(); - } - - @Test - public void testCqlNativeStorageFilter() throws IOException - { - executeCQLStatements(deleteCopyOfSomeAppTableData); - pig.setBatchOn(); - pig.registerQuery("rows = LOAD 'cql://thrift_ks/some_app?" + defaultParameters + nativeParameters + "&input_cql=select%20*%20from%20some_app%20where%20token(key)%20%3E%20%3F%20and%20token(key)%20%3C%3D%20%3F' USING CqlNativeStorage();"); - - //filter - pig.registerQuery("likes = FILTER rows by vote_type eq 'like' and rating > 5;"); - pig.registerQuery("records = FOREACH likes GENERATE TOTUPLE(TOTUPLE('key', key)),TOTUPLE(atomic_weight, created, name, percent, rating, score, vote_type);"); - pig.registerQuery("STORE records INTO 'cql://thrift_ks/copy_of_some_app?" + defaultParameters + nativeParameters + "&output_query=UPDATE+thrift_ks.copy_of_some_app+set+atomic_weight+%3D+%3F,+created+%3D+%3F,+name+%3D+%3F,+percent+%3D+%3F,+rating+%3D+%3F,+score+%3D+%3F,+vote_type+%3D+%3F' USING CqlNativeStorage();"); - pig.executeBatch(); - - Assert.assertEquals("like", getColumnValue("thrift_ks", "copy_of_some_app", "vote_type", "bar", "UTF8Type")); - Assert.assertEquals("like", getColumnValue("thrift_ks", "copy_of_some_app", "vote_type", "foo", "UTF8Type")); - String value = getColumnValue("thrift_ks", "copy_of_some_app", "vote_type", "qux", "UTF8Type"); - if (value != null) - Assert.fail(); - value = getColumnValue("thrift_ks", "copy_of_some_app", "vote_type", "baz", "UTF8Type"); - - if (value != null) - Assert.fail(); - - executeCQLStatements(deleteCopyOfSomeAppTableData); - pig.setBatchOn(); - pig.registerQuery("rows = LOAD 'cql://thrift_ks/some_app?" + defaultParameters + nativeParameters + "&input_cql=select%20*%20from%20some_app%20where%20token(key)%20%3E%20%3F%20and%20token(key)%20%3C%3D%20%3F' USING CqlNativeStorage();"); - pig.registerQuery("dislikes_extras = FILTER rows by vote_type eq 'dislike';"); - pig.registerQuery("dislikes_records = FOREACH dislikes_extras GENERATE TOTUPLE(TOTUPLE('key', key)),TOTUPLE(atomic_weight, created, name, percent, rating, score, vote_type);"); - pig.registerQuery("STORE dislikes_records INTO 'cql://thrift_ks/copy_of_some_app?" + defaultParameters + nativeParameters + "&output_query=UPDATE+thrift_ks.copy_of_some_app+set+atomic_weight+%3D+%3F,+created+%3D+%3F,+name+%3D+%3F,+percent+%3D+%3F,+rating+%3D+%3F,+score+%3D+%3F,+vote_type+%3D+%3F' USING CqlNativeStorage();"); - pig.executeBatch(); - Assert.assertEquals("dislike", getColumnValue("thrift_ks", "copy_of_some_app", "vote_type", "baz", "UTF8Type")); - Assert.assertEquals("dislike", getColumnValue("thrift_ks", "copy_of_some_app", "vote_type", "qux", "UTF8Type")); - value = getColumnValue("thrift_ks", "copy_of_some_app", "vote_type", "bar", "UTF8Type"); - if (value != null) - Assert.fail(); - value = getColumnValue("thrift_ks", "copy_of_some_app", "vote_type", "foo", "UTF8Type"); - if (value != null) - Assert.fail(); - } - - @Test - public void testCqlNativeStorageJoin() throws IOException - { - //test key types with a join - pig.registerQuery("U8 = load 'cql://thrift_ks/u8?" + defaultParameters + nativeParameters + "&input_cql=select%20*%20from%20u8%20where%20token(key)%20%3E%20%3F%20and%20token(key)%20%3C%3D%20%3F' using CqlNativeStorage();"); - pig.registerQuery("Bytes = load 'cql://thrift_ks/bytes?" + defaultParameters + nativeParameters + "&input_cql=select%20*%20from%20bytes%20where%20token(key)%20%3E%20%3F%20and%20token(key)%20%3C%3D%20%3F' using CqlNativeStorage();"); - - //cast key to chararray - pig.registerQuery("b = foreach Bytes generate (chararray)key, column1, value;"); - - //key in Bytes is a bytearray, U8 chararray - //(foo,{(x,Z)},foo,{(x,Z)}) - pig.registerQuery("a = join Bytes by key, U8 by key;"); - Iterator it = pig.openIterator("a"); - if (it.hasNext()) { - Tuple t = it.next(); - Assert.assertEquals(t.get(0), new DataByteArray("foo".getBytes())); - Assert.assertEquals(t.get(1), "x"); - Assert.assertEquals(t.get(2), new DataByteArray("Z".getBytes())); - Assert.assertEquals(t.get(3), "foo"); - Assert.assertEquals(t.get(4), "x"); - Assert.assertEquals(t.get(5), new DataByteArray("Z".getBytes())); - } - //key should now be cast into a chararray - //(foo,{(x,Z)},foo,{(x,Z)}) - pig.registerQuery("c = join b by (chararray)key, U8 by (chararray)key;"); - it = pig.openIterator("c"); - if (it.hasNext()) { - Tuple t = it.next(); - Assert.assertEquals(t.get(0), "foo"); - Assert.assertEquals(t.get(1), "x"); - Assert.assertEquals(t.get(2), new DataByteArray("Z".getBytes())); - Assert.assertEquals(t.get(3), "foo"); - Assert.assertEquals(t.get(4), "x"); - Assert.assertEquals(t.get(5), new DataByteArray("Z".getBytes())); - } - } - - @Test - public void testCqlNativeStorageCounterCF() throws IOException - { - //Test counter column family support - pig.registerQuery("CC = load 'cql://thrift_ks/cc?" + defaultParameters + nativeParameters + "&input_cql=select%20*%20from%20cc%20where%20token(key)%20%3E%20%3F%20and%20token(key)%20%3C%3D%20%3F' using CqlNativeStorage();"); - pig.registerQuery("A = foreach CC generate key, name, value;"); - pig.registerQuery("B = GROUP A BY key;"); - pig.registerQuery("total_hits = foreach B generate group, SUM(A.value);"); - //(chuck,4) - Tuple t = pig.openIterator("total_hits").next(); - Assert.assertEquals(t.get(0), "chuck"); - Assert.assertEquals(t.get(1), 4l); - } - - @Test - public void testCqlNativeStorageCompositeColumnCF() throws IOException - { - //Test CompositeType - pig.registerQuery("compo = load 'cql://thrift_ks/compo?" + defaultParameters + nativeParameters + "&input_cql=select%20*%20from%20compo%20where%20token(key)%20%3E%20%3F%20and%20token(key)%20%3C%3D%20%3F' using CqlNativeStorage();"); - pig.registerQuery("lee = filter compo by column1 == 'bruce' AND column2 == 'lee';"); - - //(kick,(bruce,lee),oww) - //(punch,(bruce,lee),ouch) - Iterator it = pig.openIterator("lee"); - int count = 0; - while (it.hasNext()) { - count ++; - Tuple t = it.next(); - Assert.assertEquals(t.get(1), "bruce"); - Assert.assertEquals(t.get(2), "lee"); - if ("kick".equals(t.get(0))) - Assert.assertEquals(t.get(3), "oww"); - else - Assert.assertEquals(t.get(3), "ouch"); - } - Assert.assertEquals(count, 2); - pig.registerQuery("night = load 'cql://thrift_ks/compo_int?" + defaultParameters + nativeParameters + "&input_cql=select%20*%20from%20compo_int%20where%20token(key)%20%3E%20%3F%20and%20token(key)%20%3C%3D%20%3F' using CqlNativeStorage();"); - pig.registerQuery("night = foreach night generate (int)column1+(double)column2/60 as hour, value as noise;"); - - //What happens at the darkest hour? - pig.registerQuery("darkest = filter night by hour > 2 and hour < 5;"); - - //(2.5,daddy?) - it = pig.openIterator("darkest"); - if (it.hasNext()) { - Tuple t = it.next(); - Assert.assertEquals(t.get(0), 2.5d); - Assert.assertEquals(t.get(1), "daddy?"); - } - pig.setBatchOn(); - pig.registerQuery("compo_int_rows = LOAD 'cql://thrift_ks/compo_int?" + defaultParameters + nativeParameters + "&input_cql=select%20*%20from%20compo_int%20where%20token(key)%20%3E%20%3F%20and%20token(key)%20%3C%3D%20%3F' using CqlNativeStorage();"); - pig.registerQuery("STORE compo_int_rows INTO 'cql://thrift_ks/compo_int_copy?" + defaultParameters + nativeParameters + "&output_query=UPDATE+thrift_ks.compo_int_copy+set+column1+%3D+%3F,+column2+%3D+%3F,+value+%3D+%3F' using CqlNativeStorage();"); - pig.executeBatch(); - pig.registerQuery("compocopy_int_rows = LOAD 'cql://thrift_ks/compo_int_copy?" + defaultParameters + nativeParameters + "&input_cql=select%20*%20from%20compo_int_copy%20where%20token(key)%20%3E%20%3F%20and%20token(key)%20%3C%3D%20%3F' using CqlNativeStorage();"); - //(clock,{((1,0),z),((1,30),zzzz),((2,30),daddy?),((6,30),coffee...)}) - it = pig.openIterator("compocopy_int_rows"); - count = 0; - if (it.hasNext()) { - Tuple t = it.next(); - Assert.assertEquals(t.get(0), "clock"); - DataBag columns = (DataBag) t.get(1); - for (Tuple t1 : columns) - { - count++; - Tuple inner = (Tuple) t1.get(0); - if ((Long) inner.get(0) == 1L && (Long) inner.get(1) == 0L) - Assert.assertEquals(t1.get(1), "z"); - else if ((Long) inner.get(0) == 1L && (Long) inner.get(1) == 30L) - Assert.assertEquals(t1.get(1), "zzzz"); - else if ((Long) inner.get(0) == 2L && (Long) inner.get(1) == 30L) - Assert.assertEquals(t1.get(1), "daddy?"); - else if ((Long) inner.get(0) == 6L && (Long) inner.get(1) == 30L) - Assert.assertEquals(t1.get(1), "coffee..."); - } - Assert.assertEquals(count, 4); - } - } - - @Test - public void testCqlNativeStorageCompositeKeyCF() throws IOException - { - //Test CompositeKey - pig.registerQuery("compokeys = load 'cql://thrift_ks/compo_key?" + defaultParameters + nativeParameters + "&input_cql=select%20*%20from%20compo_key%20where%20token(key,column1)%20%3E%20%3F%20and%20token(key,column1)%20%3C%3D%20%3F' using CqlNativeStorage();"); - pig.registerQuery("compokeys = filter compokeys by column1 == 40;"); - //(clock,40,6,coffee...) - Iterator it = pig.openIterator("compokeys"); - if (it.hasNext()) { - Tuple t = it.next(); - Assert.assertEquals(t.get(0), "clock"); - Assert.assertEquals(t.get(1), 40L); - Assert.assertEquals(t.get(2), 6L); - Assert.assertEquals(t.get(3), "coffee..."); - } - pig.setBatchOn(); - pig.registerQuery("compo_key_rows = LOAD 'cql://thrift_ks/compo_key?" + defaultParameters + nativeParameters + "&input_cql=select%20*%20from%20compo_key%20where%20token(key,column1)%20%3E%20%3F%20and%20token(key,column1)%20%3C%3D%20%3F' using CqlNativeStorage();"); - pig.registerQuery("compo_key_rows = FOREACH compo_key_rows GENERATE TOTUPLE(TOTUPLE('key',key),TOTUPLE('column1',column1),TOTUPLE('column2',column2)),TOTUPLE(value);"); - pig.registerQuery("STORE compo_key_rows INTO 'cql://thrift_ks/compo_key_copy?" + defaultParameters + nativeParameters + "&output_query=UPDATE+thrift_ks.compo_key_copy+set+value+%3D+%3F' using CqlNativeStorage();"); - pig.executeBatch(); - pig.registerQuery("compo_key_copy_rows = LOAD 'cql://thrift_ks/compo_key_copy?" + defaultParameters + nativeParameters + "&input_cql=select%20*%20from%20compo_key_copy%20where%20token(key,column1)%20%3E%20%3F%20and%20token(key,column1)%20%3C%3D%20%3F' using CqlNativeStorage();"); - //((clock,10),{(1,z)}) - //((clock,20),{(1,zzzz)}) - //((clock,30),{(2,daddy?)}) - //((clock,40),{(6,coffee...)}) - it = pig.openIterator("compo_key_copy_rows"); - int count = 0; - while (it.hasNext()) { - Tuple t = it.next(); - count ++; - if ("clock".equals(t.get(0)) && (Long) t.get(1) == 10L) - { - Assert.assertEquals(t.get(2), 1L); - Assert.assertEquals(t.get(3), "z"); - } - else if ("clock".equals(t.get(0)) && (Long) t.get(1) == 40L) - { - Assert.assertEquals(t.get(2), 6L); - Assert.assertEquals(t.get(3), "coffee..."); - } - else if ("clock".equals(t.get(0)) && (Long) t.get(1) == 20L) - { - Assert.assertEquals(t.get(2), 1L); - Assert.assertEquals(t.get(3), "zzzz"); - } - else if ("clock".equals(t.get(0)) && (Long) t.get(1) == 30L) - { - Assert.assertEquals(t.get(2), 2L); - Assert.assertEquals(t.get(3), "daddy?"); - } - } - Assert.assertEquals(4, count); - } - - private String getColumnValue(String ks, String cf, String colName, String key, String validator) throws IOException - { - Session client = getClient(); - client.execute("USE " + ks); - - String query = String.format("SELECT %s FROM %s WHERE key = '%s'", colName, cf, key); - - ResultSet rows = client.execute(query); - Row row = rows.one(); - - if (row == null || row.isNull(0)) - return null; - - return parseType(validator).getString(row.getBytesUnsafe(0)); - } -} diff --git a/test/pig/org/apache/pig/test/MiniCluster.java b/test/pig/org/apache/pig/test/MiniCluster.java deleted file mode 100644 index 95acb849e6..0000000000 --- a/test/pig/org/apache/pig/test/MiniCluster.java +++ /dev/null @@ -1,86 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.apache.pig.test; - -import java.io.File; -import java.io.FileOutputStream; -import java.io.IOException; -import java.io.OutputStream; - -import org.apache.cassandra.utils.FBUtilities; -import org.apache.hadoop.conf.Configuration; -import org.apache.hadoop.hdfs.MiniDFSCluster; -import org.apache.hadoop.mapred.MiniMRCluster; - -public class MiniCluster extends MiniGenericCluster { - private MiniMRCluster m_mr = null; - public MiniCluster() { - super(); - } - - @Override - protected void setupMiniDfsAndMrClusters() { - try { - System.setProperty("hadoop.log.dir", "build/test/logs"); - final int dataNodes = 4; // There will be 4 data nodes - final int taskTrackers = 4; // There will be 4 task tracker nodes - - // Create the configuration hadoop-site.xml file - File conf_dir = new File("build/classes/"); - conf_dir.mkdirs(); - File conf_file = new File(conf_dir, "hadoop-site.xml"); - - conf_file.delete(); - - // Builds and starts the mini dfs and mapreduce clusters - Configuration config = new Configuration(); - if (FBUtilities.isWindows()) - config.set("fs.file.impl", WindowsLocalFileSystem.class.getName()); - m_dfs = new MiniDFSCluster(config, dataNodes, true, null); - m_fileSys = m_dfs.getFileSystem(); - m_mr = new MiniMRCluster(taskTrackers, m_fileSys.getUri().toString(), 1); - - // Write the necessary config info to hadoop-site.xml - m_conf = m_mr.createJobConf(); - m_conf.setInt("mapred.submit.replication", 2); - m_conf.set("dfs.datanode.address", "0.0.0.0:0"); - m_conf.set("dfs.datanode.http.address", "0.0.0.0:0"); - m_conf.set("mapred.map.max.attempts", "2"); - m_conf.set("mapred.reduce.max.attempts", "2"); - m_conf.set("pig.jobcontrol.sleep", "100"); - try (OutputStream os = new FileOutputStream(conf_file)) - { - m_conf.writeXml(os); - } - - // Set the system properties needed by Pig - System.setProperty("cluster", m_conf.get("mapred.job.tracker")); - System.setProperty("namenode", m_conf.get("fs.default.name")); - System.setProperty("junit.hadoop.conf", conf_dir.getPath()); - } catch (IOException e) { - throw new RuntimeException(e); - } - } - - @Override - protected void shutdownMiniMrClusters() { - if (m_mr != null) - m_mr.shutdown(); - m_mr = null; - } -} diff --git a/test/pig/org/apache/pig/test/MiniGenericCluster.java b/test/pig/org/apache/pig/test/MiniGenericCluster.java deleted file mode 100644 index ac3f5bc803..0000000000 --- a/test/pig/org/apache/pig/test/MiniGenericCluster.java +++ /dev/null @@ -1,122 +0,0 @@ -/** - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.apache.pig.test; - -import java.io.*; -import java.util.Properties; - -import org.apache.hadoop.hdfs.MiniDFSCluster; - -import org.apache.hadoop.conf.Configuration; -import org.apache.hadoop.fs.FileSystem; -import org.apache.pig.backend.hadoop.datastorage.ConfigurationUtil; - -/** - * This class builds a single instance of itself with the Singleton - * design pattern. While building the single instance, it sets up a - * mini cluster that actually consists of a mini DFS cluster and a - * mini MapReduce cluster on the local machine and also sets up the - * environment for Pig to run on top of the mini cluster. - * - * This class is the base class for MiniCluster, which has slightly - * difference among different versions of hadoop. MiniCluster implementation - * is located in $PIG_HOME/shims. - */ -abstract public class MiniGenericCluster { - protected MiniDFSCluster m_dfs = null; - protected FileSystem m_fileSys = null; - protected Configuration m_conf = null; - - protected final static MiniCluster INSTANCE = new MiniCluster(); - protected static boolean isSetup = true; - - protected MiniGenericCluster() { - setupMiniDfsAndMrClusters(); - } - - abstract protected void setupMiniDfsAndMrClusters(); - - /** - * Returns the single instance of class MiniClusterBuilder that - * represents the resouces for a mini dfs cluster and a mini - * mapreduce cluster. - */ - public static MiniCluster buildCluster() { - if(! isSetup){ - INSTANCE.setupMiniDfsAndMrClusters(); - isSetup = true; - } - return INSTANCE; - } - - public void shutDown(){ - INSTANCE.shutdownMiniDfsAndMrClusters(); - } - - protected void finalize() { - shutdownMiniDfsAndMrClusters(); - } - - protected void shutdownMiniDfsAndMrClusters() { - isSetup = false; - shutdownMiniDfsClusters(); - shutdownMiniMrClusters(); - } - - protected void shutdownMiniDfsClusters() { - try { - if (m_fileSys != null) { m_fileSys.close(); } - } catch (IOException e) { - e.printStackTrace(); - } - if (m_dfs != null) { m_dfs.shutdown(); } - m_fileSys = null; - m_dfs = null; - } - - abstract protected void shutdownMiniMrClusters(); - - public Properties getProperties() { - errorIfNotSetup(); - return ConfigurationUtil.toProperties(m_conf); - } - - public Configuration getConfiguration() { - return new Configuration(m_conf); - } - - public void setProperty(String name, String value) { - errorIfNotSetup(); - m_conf.set(name, value); - } - - public FileSystem getFileSystem() { - errorIfNotSetup(); - return m_fileSys; - } - - /** - * Throw RunTimeException if isSetup is false - */ - private void errorIfNotSetup(){ - if(isSetup) - return; - String msg = "function called on MiniCluster that has been shutdown"; - throw new RuntimeException(msg); - } -} diff --git a/test/pig/org/apache/pig/test/WindowsLocalFileSystem.java b/test/pig/org/apache/pig/test/WindowsLocalFileSystem.java deleted file mode 100644 index 9193341775..0000000000 --- a/test/pig/org/apache/pig/test/WindowsLocalFileSystem.java +++ /dev/null @@ -1,62 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.apache.pig.test; - -import java.io.IOException; - -import org.apache.hadoop.fs.LocalFileSystem; -import org.apache.hadoop.fs.Path; -import org.apache.hadoop.fs.permission.FsPermission; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; - -/** - * Workaround for https://issues.apache.org/jira/browse/HADOOP-7682 used to allow the Pig-tests to run on Cygwin on - * a Windows box. This workaround was suggested by Joshua Caplan in the comments of HADOOP-7682. - */ -public final class WindowsLocalFileSystem extends LocalFileSystem -{ - private final Logger logger = LoggerFactory.getLogger(getClass()); - - public WindowsLocalFileSystem() - { - logger.warn("Using {} instead of org.apache.hadoop.fs.LocalFileSystem to avoid the problem linked to HADOOP-7682. " + - "IOException thrown when setting permissions will be swallowed.", getClass().getName()); - } - - @Override - public boolean mkdirs(Path path, FsPermission permission) throws IOException - { - boolean result = super.mkdirs(path); - setPermission(path, permission); - return result; - } - - @Override - public void setPermission(Path p, FsPermission permission) throws IOException - { - try - { - super.setPermission(p, permission); - } - catch (IOException e) - { - // Just swallow the Exception as logging it produces too much output. - } - } -}