From 533bf3f6a8c3a1802c5a88a79dba34027a60075a Mon Sep 17 00:00:00 2001 From: Jonathan Ellis Date: Fri, 19 Oct 2012 18:06:52 -0500 Subject: [PATCH] add describe_splits_ex providing improved split size estimate patch by Piotr Kolaczkowski; reviewed by jbellis for CASSANDRA-4803 --- interface/cassandra.thrift | 14 +- .../apache/cassandra/thrift/Cassandra.java | 1149 ++++++++++++++++- .../org/apache/cassandra/thrift/CfSplit.java | 549 ++++++++ .../apache/cassandra/thrift/Constants.java | 2 +- .../hadoop/ColumnFamilyInputFormat.java | 36 +- .../hadoop/ColumnFamilyRecordReader.java | 11 +- .../cassandra/hadoop/ColumnFamilySplit.java | 14 +- .../cassandra/service/StorageService.java | 49 +- .../cassandra/thrift/CassandraServer.java | 31 +- 9 files changed, 1801 insertions(+), 54 deletions(-) create mode 100644 interface/thrift/gen-java/org/apache/cassandra/thrift/CfSplit.java diff --git a/interface/cassandra.thrift b/interface/cassandra.thrift index 6df9628f7a..bf73c4f543 100644 --- a/interface/cassandra.thrift +++ b/interface/cassandra.thrift @@ -55,7 +55,7 @@ namespace rb CassandraThrift # An effort should be made not to break forward-client-compatibility either # (e.g. one should avoid removing obsolete fields from the IDL), but no # guarantees in this respect are made by the Cassandra project. -const string VERSION = "19.32.0" +const string VERSION = "19.33.0" # @@ -501,6 +501,12 @@ struct CqlPreparedResult { 4: optional list variable_names } +/** Represents input splits used by hadoop ColumnFamilyRecordReaders */ +struct CfSplit { + 1: required string start_token, + 2: required string end_token, + 3: required i64 row_count +} service Cassandra { # auth methods @@ -709,6 +715,12 @@ service Cassandra { 4:required i32 keys_per_split) throws (1:InvalidRequestException ire), + list describe_splits_ex(1:required string cfName, + 2:required string start_token, + 3:required string end_token, + 4:required i32 keys_per_split) + throws (1:InvalidRequestException ire), + /** adds a column family. returns the new schema id. */ string system_add_column_family(1:required CfDef cf_def) throws (1:InvalidRequestException ire, 2:SchemaDisagreementException sde), diff --git a/interface/thrift/gen-java/org/apache/cassandra/thrift/Cassandra.java b/interface/thrift/gen-java/org/apache/cassandra/thrift/Cassandra.java index 1c6ec690ad..c759610982 100644 --- a/interface/thrift/gen-java/org/apache/cassandra/thrift/Cassandra.java +++ b/interface/thrift/gen-java/org/apache/cassandra/thrift/Cassandra.java @@ -273,6 +273,8 @@ public class Cassandra { */ public List describe_splits(String cfName, String start_token, String end_token, int keys_per_split) throws InvalidRequestException, org.apache.thrift.TException; + public List describe_splits_ex(String cfName, String start_token, String end_token, int keys_per_split) throws InvalidRequestException, org.apache.thrift.TException; + /** * adds a column family. returns the new schema id. * @@ -402,6 +404,8 @@ public class Cassandra { public void describe_splits(String cfName, String start_token, String end_token, int keys_per_split, org.apache.thrift.async.AsyncMethodCallback resultHandler) throws org.apache.thrift.TException; + public void describe_splits_ex(String cfName, String start_token, String end_token, int keys_per_split, org.apache.thrift.async.AsyncMethodCallback resultHandler) throws org.apache.thrift.TException; + public void system_add_column_family(CfDef cf_def, org.apache.thrift.async.AsyncMethodCallback resultHandler) throws org.apache.thrift.TException; public void system_drop_column_family(String column_family, org.apache.thrift.async.AsyncMethodCallback resultHandler) throws org.apache.thrift.TException; @@ -1208,6 +1212,35 @@ public class Cassandra { throw new org.apache.thrift.TApplicationException(org.apache.thrift.TApplicationException.MISSING_RESULT, "describe_splits failed: unknown result"); } + public List describe_splits_ex(String cfName, String start_token, String end_token, int keys_per_split) throws InvalidRequestException, org.apache.thrift.TException + { + send_describe_splits_ex(cfName, start_token, end_token, keys_per_split); + return recv_describe_splits_ex(); + } + + public void send_describe_splits_ex(String cfName, String start_token, String end_token, int keys_per_split) throws org.apache.thrift.TException + { + describe_splits_ex_args args = new describe_splits_ex_args(); + args.setCfName(cfName); + args.setStart_token(start_token); + args.setEnd_token(end_token); + args.setKeys_per_split(keys_per_split); + sendBase("describe_splits_ex", args); + } + + public List recv_describe_splits_ex() throws InvalidRequestException, org.apache.thrift.TException + { + describe_splits_ex_result result = new describe_splits_ex_result(); + receiveBase(result, "describe_splits_ex"); + if (result.isSetSuccess()) { + return result.success; + } + if (result.ire != null) { + throw result.ire; + } + throw new org.apache.thrift.TApplicationException(org.apache.thrift.TApplicationException.MISSING_RESULT, "describe_splits_ex failed: unknown result"); + } + public String system_add_column_family(CfDef cf_def) throws InvalidRequestException, SchemaDisagreementException, org.apache.thrift.TException { send_system_add_column_family(cf_def); @@ -2447,6 +2480,47 @@ public class Cassandra { } } + public void describe_splits_ex(String cfName, String start_token, String end_token, int keys_per_split, org.apache.thrift.async.AsyncMethodCallback resultHandler) throws org.apache.thrift.TException { + checkReady(); + describe_splits_ex_call method_call = new describe_splits_ex_call(cfName, start_token, end_token, keys_per_split, resultHandler, this, ___protocolFactory, ___transport); + this.___currentMethod = method_call; + ___manager.call(method_call); + } + + public static class describe_splits_ex_call extends org.apache.thrift.async.TAsyncMethodCall { + private String cfName; + private String start_token; + private String end_token; + private int keys_per_split; + public describe_splits_ex_call(String cfName, String start_token, String end_token, int keys_per_split, org.apache.thrift.async.AsyncMethodCallback resultHandler, org.apache.thrift.async.TAsyncClient client, org.apache.thrift.protocol.TProtocolFactory protocolFactory, org.apache.thrift.transport.TNonblockingTransport transport) throws org.apache.thrift.TException { + super(client, protocolFactory, transport, resultHandler, false); + this.cfName = cfName; + this.start_token = start_token; + this.end_token = end_token; + this.keys_per_split = keys_per_split; + } + + public void write_args(org.apache.thrift.protocol.TProtocol prot) throws org.apache.thrift.TException { + prot.writeMessageBegin(new org.apache.thrift.protocol.TMessage("describe_splits_ex", org.apache.thrift.protocol.TMessageType.CALL, 0)); + describe_splits_ex_args args = new describe_splits_ex_args(); + args.setCfName(cfName); + args.setStart_token(start_token); + args.setEnd_token(end_token); + args.setKeys_per_split(keys_per_split); + args.write(prot); + prot.writeMessageEnd(); + } + + public List getResult() throws InvalidRequestException, org.apache.thrift.TException { + if (getState() != org.apache.thrift.async.TAsyncMethodCall.State.RESPONSE_READ) { + throw new IllegalStateException("Method call not finished!"); + } + org.apache.thrift.transport.TMemoryInputTransport memoryTransport = new org.apache.thrift.transport.TMemoryInputTransport(getFrameBuffer().array()); + org.apache.thrift.protocol.TProtocol prot = client.getProtocolFactory().getProtocol(memoryTransport); + return (new Client(prot)).recv_describe_splits_ex(); + } + } + public void system_add_column_family(CfDef cf_def, org.apache.thrift.async.AsyncMethodCallback resultHandler) throws org.apache.thrift.TException { checkReady(); system_add_column_family_call method_call = new system_add_column_family_call(cf_def, resultHandler, this, ___protocolFactory, ___transport); @@ -2815,6 +2889,7 @@ public class Cassandra { processMap.put("describe_snitch", new describe_snitch()); processMap.put("describe_keyspace", new describe_keyspace()); processMap.put("describe_splits", new describe_splits()); + processMap.put("describe_splits_ex", new describe_splits_ex()); processMap.put("system_add_column_family", new system_add_column_family()); processMap.put("system_drop_column_family", new system_drop_column_family()); processMap.put("system_add_keyspace", new system_add_keyspace()); @@ -3395,6 +3470,26 @@ public class Cassandra { } } + private static class describe_splits_ex extends org.apache.thrift.ProcessFunction { + public describe_splits_ex() { + super("describe_splits_ex"); + } + + protected describe_splits_ex_args getEmptyArgsInstance() { + return new describe_splits_ex_args(); + } + + protected describe_splits_ex_result getResult(I iface, describe_splits_ex_args args) throws org.apache.thrift.TException { + describe_splits_ex_result result = new describe_splits_ex_result(); + try { + result.success = iface.describe_splits_ex(args.cfName, args.start_token, args.end_token, args.keys_per_split); + } catch (InvalidRequestException ire) { + result.ire = ire; + } + return result; + } + } + private static class system_add_column_family extends org.apache.thrift.ProcessFunction { public system_add_column_family() { super("system_add_column_family"); @@ -17817,6 +17912,8 @@ public class Cassandra { private void readObject(java.io.ObjectInputStream in) throws java.io.IOException, ClassNotFoundException { try { + // it doesn't seem like you should have to do this, but java serialization is wacky, and doesn't call the default constructor. + __isset_bit_vector = new BitSet(1); read(new org.apache.thrift.protocol.TCompactProtocol(new org.apache.thrift.transport.TIOStreamTransport(in))); } catch (org.apache.thrift.TException te) { throw new java.io.IOException(te); @@ -27710,6 +27807,1040 @@ public class Cassandra { } + public static class describe_splits_ex_args implements org.apache.thrift.TBase, java.io.Serializable, Cloneable { + private static final org.apache.thrift.protocol.TStruct STRUCT_DESC = new org.apache.thrift.protocol.TStruct("describe_splits_ex_args"); + + private static final org.apache.thrift.protocol.TField CF_NAME_FIELD_DESC = new org.apache.thrift.protocol.TField("cfName", org.apache.thrift.protocol.TType.STRING, (short)1); + private static final org.apache.thrift.protocol.TField START_TOKEN_FIELD_DESC = new org.apache.thrift.protocol.TField("start_token", org.apache.thrift.protocol.TType.STRING, (short)2); + private static final org.apache.thrift.protocol.TField END_TOKEN_FIELD_DESC = new org.apache.thrift.protocol.TField("end_token", org.apache.thrift.protocol.TType.STRING, (short)3); + private static final org.apache.thrift.protocol.TField KEYS_PER_SPLIT_FIELD_DESC = new org.apache.thrift.protocol.TField("keys_per_split", org.apache.thrift.protocol.TType.I32, (short)4); + + public String cfName; // required + public String start_token; // required + public String end_token; // required + public int keys_per_split; // required + + /** The set of fields this struct contains, along with convenience methods for finding and manipulating them. */ + public enum _Fields implements org.apache.thrift.TFieldIdEnum { + CF_NAME((short)1, "cfName"), + START_TOKEN((short)2, "start_token"), + END_TOKEN((short)3, "end_token"), + KEYS_PER_SPLIT((short)4, "keys_per_split"); + + private static final Map byName = new HashMap(); + + static { + for (_Fields field : EnumSet.allOf(_Fields.class)) { + byName.put(field.getFieldName(), field); + } + } + + /** + * Find the _Fields constant that matches fieldId, or null if its not found. + */ + public static _Fields findByThriftId(int fieldId) { + switch(fieldId) { + case 1: // CF_NAME + return CF_NAME; + case 2: // START_TOKEN + return START_TOKEN; + case 3: // END_TOKEN + return END_TOKEN; + case 4: // KEYS_PER_SPLIT + return KEYS_PER_SPLIT; + default: + return null; + } + } + + /** + * Find the _Fields constant that matches fieldId, throwing an exception + * if it is not found. + */ + public static _Fields findByThriftIdOrThrow(int fieldId) { + _Fields fields = findByThriftId(fieldId); + if (fields == null) throw new IllegalArgumentException("Field " + fieldId + " doesn't exist!"); + return fields; + } + + /** + * Find the _Fields constant that matches name, or null if its not found. + */ + public static _Fields findByName(String name) { + return byName.get(name); + } + + private final short _thriftId; + private final String _fieldName; + + _Fields(short thriftId, String fieldName) { + _thriftId = thriftId; + _fieldName = fieldName; + } + + public short getThriftFieldId() { + return _thriftId; + } + + public String getFieldName() { + return _fieldName; + } + } + + // isset id assignments + private static final int __KEYS_PER_SPLIT_ISSET_ID = 0; + private BitSet __isset_bit_vector = new BitSet(1); + + public static final Map<_Fields, org.apache.thrift.meta_data.FieldMetaData> metaDataMap; + static { + Map<_Fields, org.apache.thrift.meta_data.FieldMetaData> tmpMap = new EnumMap<_Fields, org.apache.thrift.meta_data.FieldMetaData>(_Fields.class); + tmpMap.put(_Fields.CF_NAME, new org.apache.thrift.meta_data.FieldMetaData("cfName", org.apache.thrift.TFieldRequirementType.REQUIRED, + new org.apache.thrift.meta_data.FieldValueMetaData(org.apache.thrift.protocol.TType.STRING))); + tmpMap.put(_Fields.START_TOKEN, new org.apache.thrift.meta_data.FieldMetaData("start_token", org.apache.thrift.TFieldRequirementType.REQUIRED, + new org.apache.thrift.meta_data.FieldValueMetaData(org.apache.thrift.protocol.TType.STRING))); + tmpMap.put(_Fields.END_TOKEN, new org.apache.thrift.meta_data.FieldMetaData("end_token", org.apache.thrift.TFieldRequirementType.REQUIRED, + new org.apache.thrift.meta_data.FieldValueMetaData(org.apache.thrift.protocol.TType.STRING))); + tmpMap.put(_Fields.KEYS_PER_SPLIT, new org.apache.thrift.meta_data.FieldMetaData("keys_per_split", org.apache.thrift.TFieldRequirementType.REQUIRED, + new org.apache.thrift.meta_data.FieldValueMetaData(org.apache.thrift.protocol.TType.I32))); + metaDataMap = Collections.unmodifiableMap(tmpMap); + org.apache.thrift.meta_data.FieldMetaData.addStructMetaDataMap(describe_splits_ex_args.class, metaDataMap); + } + + public describe_splits_ex_args() { + } + + public describe_splits_ex_args( + String cfName, + String start_token, + String end_token, + int keys_per_split) + { + this(); + this.cfName = cfName; + this.start_token = start_token; + this.end_token = end_token; + this.keys_per_split = keys_per_split; + setKeys_per_splitIsSet(true); + } + + /** + * Performs a deep copy on other. + */ + public describe_splits_ex_args(describe_splits_ex_args other) { + __isset_bit_vector.clear(); + __isset_bit_vector.or(other.__isset_bit_vector); + if (other.isSetCfName()) { + this.cfName = other.cfName; + } + if (other.isSetStart_token()) { + this.start_token = other.start_token; + } + if (other.isSetEnd_token()) { + this.end_token = other.end_token; + } + this.keys_per_split = other.keys_per_split; + } + + public describe_splits_ex_args deepCopy() { + return new describe_splits_ex_args(this); + } + + @Override + public void clear() { + this.cfName = null; + this.start_token = null; + this.end_token = null; + setKeys_per_splitIsSet(false); + this.keys_per_split = 0; + } + + public String getCfName() { + return this.cfName; + } + + public describe_splits_ex_args setCfName(String cfName) { + this.cfName = cfName; + return this; + } + + public void unsetCfName() { + this.cfName = null; + } + + /** Returns true if field cfName is set (has been assigned a value) and false otherwise */ + public boolean isSetCfName() { + return this.cfName != null; + } + + public void setCfNameIsSet(boolean value) { + if (!value) { + this.cfName = null; + } + } + + public String getStart_token() { + return this.start_token; + } + + public describe_splits_ex_args setStart_token(String start_token) { + this.start_token = start_token; + return this; + } + + public void unsetStart_token() { + this.start_token = null; + } + + /** Returns true if field start_token is set (has been assigned a value) and false otherwise */ + public boolean isSetStart_token() { + return this.start_token != null; + } + + public void setStart_tokenIsSet(boolean value) { + if (!value) { + this.start_token = null; + } + } + + public String getEnd_token() { + return this.end_token; + } + + public describe_splits_ex_args setEnd_token(String end_token) { + this.end_token = end_token; + return this; + } + + public void unsetEnd_token() { + this.end_token = null; + } + + /** Returns true if field end_token is set (has been assigned a value) and false otherwise */ + public boolean isSetEnd_token() { + return this.end_token != null; + } + + public void setEnd_tokenIsSet(boolean value) { + if (!value) { + this.end_token = null; + } + } + + public int getKeys_per_split() { + return this.keys_per_split; + } + + public describe_splits_ex_args setKeys_per_split(int keys_per_split) { + this.keys_per_split = keys_per_split; + setKeys_per_splitIsSet(true); + return this; + } + + public void unsetKeys_per_split() { + __isset_bit_vector.clear(__KEYS_PER_SPLIT_ISSET_ID); + } + + /** Returns true if field keys_per_split is set (has been assigned a value) and false otherwise */ + public boolean isSetKeys_per_split() { + return __isset_bit_vector.get(__KEYS_PER_SPLIT_ISSET_ID); + } + + public void setKeys_per_splitIsSet(boolean value) { + __isset_bit_vector.set(__KEYS_PER_SPLIT_ISSET_ID, value); + } + + public void setFieldValue(_Fields field, Object value) { + switch (field) { + case CF_NAME: + if (value == null) { + unsetCfName(); + } else { + setCfName((String)value); + } + break; + + case START_TOKEN: + if (value == null) { + unsetStart_token(); + } else { + setStart_token((String)value); + } + break; + + case END_TOKEN: + if (value == null) { + unsetEnd_token(); + } else { + setEnd_token((String)value); + } + break; + + case KEYS_PER_SPLIT: + if (value == null) { + unsetKeys_per_split(); + } else { + setKeys_per_split((Integer)value); + } + break; + + } + } + + public Object getFieldValue(_Fields field) { + switch (field) { + case CF_NAME: + return getCfName(); + + case START_TOKEN: + return getStart_token(); + + case END_TOKEN: + return getEnd_token(); + + case KEYS_PER_SPLIT: + return Integer.valueOf(getKeys_per_split()); + + } + throw new IllegalStateException(); + } + + /** Returns true if field corresponding to fieldID is set (has been assigned a value) and false otherwise */ + public boolean isSet(_Fields field) { + if (field == null) { + throw new IllegalArgumentException(); + } + + switch (field) { + case CF_NAME: + return isSetCfName(); + case START_TOKEN: + return isSetStart_token(); + case END_TOKEN: + return isSetEnd_token(); + case KEYS_PER_SPLIT: + return isSetKeys_per_split(); + } + throw new IllegalStateException(); + } + + @Override + public boolean equals(Object that) { + if (that == null) + return false; + if (that instanceof describe_splits_ex_args) + return this.equals((describe_splits_ex_args)that); + return false; + } + + public boolean equals(describe_splits_ex_args that) { + if (that == null) + return false; + + boolean this_present_cfName = true && this.isSetCfName(); + boolean that_present_cfName = true && that.isSetCfName(); + if (this_present_cfName || that_present_cfName) { + if (!(this_present_cfName && that_present_cfName)) + return false; + if (!this.cfName.equals(that.cfName)) + return false; + } + + boolean this_present_start_token = true && this.isSetStart_token(); + boolean that_present_start_token = true && that.isSetStart_token(); + if (this_present_start_token || that_present_start_token) { + if (!(this_present_start_token && that_present_start_token)) + return false; + if (!this.start_token.equals(that.start_token)) + return false; + } + + boolean this_present_end_token = true && this.isSetEnd_token(); + boolean that_present_end_token = true && that.isSetEnd_token(); + if (this_present_end_token || that_present_end_token) { + if (!(this_present_end_token && that_present_end_token)) + return false; + if (!this.end_token.equals(that.end_token)) + return false; + } + + boolean this_present_keys_per_split = true; + boolean that_present_keys_per_split = true; + if (this_present_keys_per_split || that_present_keys_per_split) { + if (!(this_present_keys_per_split && that_present_keys_per_split)) + return false; + if (this.keys_per_split != that.keys_per_split) + return false; + } + + return true; + } + + @Override + public int hashCode() { + HashCodeBuilder builder = new HashCodeBuilder(); + + boolean present_cfName = true && (isSetCfName()); + builder.append(present_cfName); + if (present_cfName) + builder.append(cfName); + + boolean present_start_token = true && (isSetStart_token()); + builder.append(present_start_token); + if (present_start_token) + builder.append(start_token); + + boolean present_end_token = true && (isSetEnd_token()); + builder.append(present_end_token); + if (present_end_token) + builder.append(end_token); + + boolean present_keys_per_split = true; + builder.append(present_keys_per_split); + if (present_keys_per_split) + builder.append(keys_per_split); + + return builder.toHashCode(); + } + + public int compareTo(describe_splits_ex_args other) { + if (!getClass().equals(other.getClass())) { + return getClass().getName().compareTo(other.getClass().getName()); + } + + int lastComparison = 0; + describe_splits_ex_args typedOther = (describe_splits_ex_args)other; + + lastComparison = Boolean.valueOf(isSetCfName()).compareTo(typedOther.isSetCfName()); + if (lastComparison != 0) { + return lastComparison; + } + if (isSetCfName()) { + lastComparison = org.apache.thrift.TBaseHelper.compareTo(this.cfName, typedOther.cfName); + if (lastComparison != 0) { + return lastComparison; + } + } + lastComparison = Boolean.valueOf(isSetStart_token()).compareTo(typedOther.isSetStart_token()); + if (lastComparison != 0) { + return lastComparison; + } + if (isSetStart_token()) { + lastComparison = org.apache.thrift.TBaseHelper.compareTo(this.start_token, typedOther.start_token); + if (lastComparison != 0) { + return lastComparison; + } + } + lastComparison = Boolean.valueOf(isSetEnd_token()).compareTo(typedOther.isSetEnd_token()); + if (lastComparison != 0) { + return lastComparison; + } + if (isSetEnd_token()) { + lastComparison = org.apache.thrift.TBaseHelper.compareTo(this.end_token, typedOther.end_token); + if (lastComparison != 0) { + return lastComparison; + } + } + lastComparison = Boolean.valueOf(isSetKeys_per_split()).compareTo(typedOther.isSetKeys_per_split()); + if (lastComparison != 0) { + return lastComparison; + } + if (isSetKeys_per_split()) { + lastComparison = org.apache.thrift.TBaseHelper.compareTo(this.keys_per_split, typedOther.keys_per_split); + if (lastComparison != 0) { + return lastComparison; + } + } + return 0; + } + + public _Fields fieldForId(int fieldId) { + return _Fields.findByThriftId(fieldId); + } + + public void read(org.apache.thrift.protocol.TProtocol iprot) throws org.apache.thrift.TException { + org.apache.thrift.protocol.TField field; + iprot.readStructBegin(); + while (true) + { + field = iprot.readFieldBegin(); + if (field.type == org.apache.thrift.protocol.TType.STOP) { + break; + } + switch (field.id) { + case 1: // CF_NAME + if (field.type == org.apache.thrift.protocol.TType.STRING) { + this.cfName = iprot.readString(); + } else { + org.apache.thrift.protocol.TProtocolUtil.skip(iprot, field.type); + } + break; + case 2: // START_TOKEN + if (field.type == org.apache.thrift.protocol.TType.STRING) { + this.start_token = iprot.readString(); + } else { + org.apache.thrift.protocol.TProtocolUtil.skip(iprot, field.type); + } + break; + case 3: // END_TOKEN + if (field.type == org.apache.thrift.protocol.TType.STRING) { + this.end_token = iprot.readString(); + } else { + org.apache.thrift.protocol.TProtocolUtil.skip(iprot, field.type); + } + break; + case 4: // KEYS_PER_SPLIT + if (field.type == org.apache.thrift.protocol.TType.I32) { + this.keys_per_split = iprot.readI32(); + setKeys_per_splitIsSet(true); + } else { + org.apache.thrift.protocol.TProtocolUtil.skip(iprot, field.type); + } + break; + default: + org.apache.thrift.protocol.TProtocolUtil.skip(iprot, field.type); + } + iprot.readFieldEnd(); + } + iprot.readStructEnd(); + + // check for required fields of primitive type, which can't be checked in the validate method + if (!isSetKeys_per_split()) { + throw new org.apache.thrift.protocol.TProtocolException("Required field 'keys_per_split' was not found in serialized data! Struct: " + toString()); + } + validate(); + } + + public void write(org.apache.thrift.protocol.TProtocol oprot) throws org.apache.thrift.TException { + validate(); + + oprot.writeStructBegin(STRUCT_DESC); + if (this.cfName != null) { + oprot.writeFieldBegin(CF_NAME_FIELD_DESC); + oprot.writeString(this.cfName); + oprot.writeFieldEnd(); + } + if (this.start_token != null) { + oprot.writeFieldBegin(START_TOKEN_FIELD_DESC); + oprot.writeString(this.start_token); + oprot.writeFieldEnd(); + } + if (this.end_token != null) { + oprot.writeFieldBegin(END_TOKEN_FIELD_DESC); + oprot.writeString(this.end_token); + oprot.writeFieldEnd(); + } + oprot.writeFieldBegin(KEYS_PER_SPLIT_FIELD_DESC); + oprot.writeI32(this.keys_per_split); + oprot.writeFieldEnd(); + oprot.writeFieldStop(); + oprot.writeStructEnd(); + } + + @Override + public String toString() { + StringBuilder sb = new StringBuilder("describe_splits_ex_args("); + boolean first = true; + + sb.append("cfName:"); + if (this.cfName == null) { + sb.append("null"); + } else { + sb.append(this.cfName); + } + first = false; + if (!first) sb.append(", "); + sb.append("start_token:"); + if (this.start_token == null) { + sb.append("null"); + } else { + sb.append(this.start_token); + } + first = false; + if (!first) sb.append(", "); + sb.append("end_token:"); + if (this.end_token == null) { + sb.append("null"); + } else { + sb.append(this.end_token); + } + first = false; + if (!first) sb.append(", "); + sb.append("keys_per_split:"); + sb.append(this.keys_per_split); + first = false; + sb.append(")"); + return sb.toString(); + } + + public void validate() throws org.apache.thrift.TException { + // check for required fields + if (cfName == null) { + throw new org.apache.thrift.protocol.TProtocolException("Required field 'cfName' was not present! Struct: " + toString()); + } + if (start_token == null) { + throw new org.apache.thrift.protocol.TProtocolException("Required field 'start_token' was not present! Struct: " + toString()); + } + if (end_token == null) { + throw new org.apache.thrift.protocol.TProtocolException("Required field 'end_token' was not present! Struct: " + toString()); + } + // alas, we cannot check 'keys_per_split' because it's a primitive and you chose the non-beans generator. + } + + private void writeObject(java.io.ObjectOutputStream out) throws java.io.IOException { + try { + write(new org.apache.thrift.protocol.TCompactProtocol(new org.apache.thrift.transport.TIOStreamTransport(out))); + } catch (org.apache.thrift.TException te) { + throw new java.io.IOException(te); + } + } + + private void readObject(java.io.ObjectInputStream in) throws java.io.IOException, ClassNotFoundException { + try { + // it doesn't seem like you should have to do this, but java serialization is wacky, and doesn't call the default constructor. + __isset_bit_vector = new BitSet(1); + read(new org.apache.thrift.protocol.TCompactProtocol(new org.apache.thrift.transport.TIOStreamTransport(in))); + } catch (org.apache.thrift.TException te) { + throw new java.io.IOException(te); + } + } + + } + + public static class describe_splits_ex_result implements org.apache.thrift.TBase, java.io.Serializable, Cloneable { + private static final org.apache.thrift.protocol.TStruct STRUCT_DESC = new org.apache.thrift.protocol.TStruct("describe_splits_ex_result"); + + private static final org.apache.thrift.protocol.TField SUCCESS_FIELD_DESC = new org.apache.thrift.protocol.TField("success", org.apache.thrift.protocol.TType.LIST, (short)0); + private static final org.apache.thrift.protocol.TField IRE_FIELD_DESC = new org.apache.thrift.protocol.TField("ire", org.apache.thrift.protocol.TType.STRUCT, (short)1); + + public List success; // required + public InvalidRequestException ire; // required + + /** The set of fields this struct contains, along with convenience methods for finding and manipulating them. */ + public enum _Fields implements org.apache.thrift.TFieldIdEnum { + SUCCESS((short)0, "success"), + IRE((short)1, "ire"); + + private static final Map byName = new HashMap(); + + static { + for (_Fields field : EnumSet.allOf(_Fields.class)) { + byName.put(field.getFieldName(), field); + } + } + + /** + * Find the _Fields constant that matches fieldId, or null if its not found. + */ + public static _Fields findByThriftId(int fieldId) { + switch(fieldId) { + case 0: // SUCCESS + return SUCCESS; + case 1: // IRE + return IRE; + default: + return null; + } + } + + /** + * Find the _Fields constant that matches fieldId, throwing an exception + * if it is not found. + */ + public static _Fields findByThriftIdOrThrow(int fieldId) { + _Fields fields = findByThriftId(fieldId); + if (fields == null) throw new IllegalArgumentException("Field " + fieldId + " doesn't exist!"); + return fields; + } + + /** + * Find the _Fields constant that matches name, or null if its not found. + */ + public static _Fields findByName(String name) { + return byName.get(name); + } + + private final short _thriftId; + private final String _fieldName; + + _Fields(short thriftId, String fieldName) { + _thriftId = thriftId; + _fieldName = fieldName; + } + + public short getThriftFieldId() { + return _thriftId; + } + + public String getFieldName() { + return _fieldName; + } + } + + // isset id assignments + + public static final Map<_Fields, org.apache.thrift.meta_data.FieldMetaData> metaDataMap; + static { + Map<_Fields, org.apache.thrift.meta_data.FieldMetaData> tmpMap = new EnumMap<_Fields, org.apache.thrift.meta_data.FieldMetaData>(_Fields.class); + tmpMap.put(_Fields.SUCCESS, new org.apache.thrift.meta_data.FieldMetaData("success", org.apache.thrift.TFieldRequirementType.DEFAULT, + new org.apache.thrift.meta_data.ListMetaData(org.apache.thrift.protocol.TType.LIST, + new org.apache.thrift.meta_data.StructMetaData(org.apache.thrift.protocol.TType.STRUCT, CfSplit.class)))); + tmpMap.put(_Fields.IRE, new org.apache.thrift.meta_data.FieldMetaData("ire", org.apache.thrift.TFieldRequirementType.DEFAULT, + new org.apache.thrift.meta_data.FieldValueMetaData(org.apache.thrift.protocol.TType.STRUCT))); + metaDataMap = Collections.unmodifiableMap(tmpMap); + org.apache.thrift.meta_data.FieldMetaData.addStructMetaDataMap(describe_splits_ex_result.class, metaDataMap); + } + + public describe_splits_ex_result() { + } + + public describe_splits_ex_result( + List success, + InvalidRequestException ire) + { + this(); + this.success = success; + this.ire = ire; + } + + /** + * Performs a deep copy on other. + */ + public describe_splits_ex_result(describe_splits_ex_result other) { + if (other.isSetSuccess()) { + List __this__success = new ArrayList(); + for (CfSplit other_element : other.success) { + __this__success.add(new CfSplit(other_element)); + } + this.success = __this__success; + } + if (other.isSetIre()) { + this.ire = new InvalidRequestException(other.ire); + } + } + + public describe_splits_ex_result deepCopy() { + return new describe_splits_ex_result(this); + } + + @Override + public void clear() { + this.success = null; + this.ire = null; + } + + public int getSuccessSize() { + return (this.success == null) ? 0 : this.success.size(); + } + + public java.util.Iterator getSuccessIterator() { + return (this.success == null) ? null : this.success.iterator(); + } + + public void addToSuccess(CfSplit elem) { + if (this.success == null) { + this.success = new ArrayList(); + } + this.success.add(elem); + } + + public List getSuccess() { + return this.success; + } + + public describe_splits_ex_result setSuccess(List success) { + this.success = success; + return this; + } + + public void unsetSuccess() { + this.success = null; + } + + /** Returns true if field success is set (has been assigned a value) and false otherwise */ + public boolean isSetSuccess() { + return this.success != null; + } + + public void setSuccessIsSet(boolean value) { + if (!value) { + this.success = null; + } + } + + public InvalidRequestException getIre() { + return this.ire; + } + + public describe_splits_ex_result setIre(InvalidRequestException ire) { + this.ire = ire; + return this; + } + + public void unsetIre() { + this.ire = null; + } + + /** Returns true if field ire is set (has been assigned a value) and false otherwise */ + public boolean isSetIre() { + return this.ire != null; + } + + public void setIreIsSet(boolean value) { + if (!value) { + this.ire = null; + } + } + + public void setFieldValue(_Fields field, Object value) { + switch (field) { + case SUCCESS: + if (value == null) { + unsetSuccess(); + } else { + setSuccess((List)value); + } + break; + + case IRE: + if (value == null) { + unsetIre(); + } else { + setIre((InvalidRequestException)value); + } + break; + + } + } + + public Object getFieldValue(_Fields field) { + switch (field) { + case SUCCESS: + return getSuccess(); + + case IRE: + return getIre(); + + } + throw new IllegalStateException(); + } + + /** Returns true if field corresponding to fieldID is set (has been assigned a value) and false otherwise */ + public boolean isSet(_Fields field) { + if (field == null) { + throw new IllegalArgumentException(); + } + + switch (field) { + case SUCCESS: + return isSetSuccess(); + case IRE: + return isSetIre(); + } + throw new IllegalStateException(); + } + + @Override + public boolean equals(Object that) { + if (that == null) + return false; + if (that instanceof describe_splits_ex_result) + return this.equals((describe_splits_ex_result)that); + return false; + } + + public boolean equals(describe_splits_ex_result that) { + if (that == null) + return false; + + boolean this_present_success = true && this.isSetSuccess(); + boolean that_present_success = true && that.isSetSuccess(); + if (this_present_success || that_present_success) { + if (!(this_present_success && that_present_success)) + return false; + if (!this.success.equals(that.success)) + return false; + } + + boolean this_present_ire = true && this.isSetIre(); + boolean that_present_ire = true && that.isSetIre(); + if (this_present_ire || that_present_ire) { + if (!(this_present_ire && that_present_ire)) + return false; + if (!this.ire.equals(that.ire)) + return false; + } + + return true; + } + + @Override + public int hashCode() { + HashCodeBuilder builder = new HashCodeBuilder(); + + boolean present_success = true && (isSetSuccess()); + builder.append(present_success); + if (present_success) + builder.append(success); + + boolean present_ire = true && (isSetIre()); + builder.append(present_ire); + if (present_ire) + builder.append(ire); + + return builder.toHashCode(); + } + + public int compareTo(describe_splits_ex_result other) { + if (!getClass().equals(other.getClass())) { + return getClass().getName().compareTo(other.getClass().getName()); + } + + int lastComparison = 0; + describe_splits_ex_result typedOther = (describe_splits_ex_result)other; + + lastComparison = Boolean.valueOf(isSetSuccess()).compareTo(typedOther.isSetSuccess()); + if (lastComparison != 0) { + return lastComparison; + } + if (isSetSuccess()) { + lastComparison = org.apache.thrift.TBaseHelper.compareTo(this.success, typedOther.success); + if (lastComparison != 0) { + return lastComparison; + } + } + lastComparison = Boolean.valueOf(isSetIre()).compareTo(typedOther.isSetIre()); + if (lastComparison != 0) { + return lastComparison; + } + if (isSetIre()) { + lastComparison = org.apache.thrift.TBaseHelper.compareTo(this.ire, typedOther.ire); + if (lastComparison != 0) { + return lastComparison; + } + } + return 0; + } + + public _Fields fieldForId(int fieldId) { + return _Fields.findByThriftId(fieldId); + } + + public void read(org.apache.thrift.protocol.TProtocol iprot) throws org.apache.thrift.TException { + org.apache.thrift.protocol.TField field; + iprot.readStructBegin(); + while (true) + { + field = iprot.readFieldBegin(); + if (field.type == org.apache.thrift.protocol.TType.STOP) { + break; + } + switch (field.id) { + case 0: // SUCCESS + if (field.type == org.apache.thrift.protocol.TType.LIST) { + { + org.apache.thrift.protocol.TList _list173 = iprot.readListBegin(); + this.success = new ArrayList(_list173.size); + for (int _i174 = 0; _i174 < _list173.size; ++_i174) + { + CfSplit _elem175; // required + _elem175 = new CfSplit(); + _elem175.read(iprot); + this.success.add(_elem175); + } + iprot.readListEnd(); + } + } else { + org.apache.thrift.protocol.TProtocolUtil.skip(iprot, field.type); + } + break; + case 1: // IRE + if (field.type == org.apache.thrift.protocol.TType.STRUCT) { + this.ire = new InvalidRequestException(); + this.ire.read(iprot); + } else { + org.apache.thrift.protocol.TProtocolUtil.skip(iprot, field.type); + } + break; + default: + org.apache.thrift.protocol.TProtocolUtil.skip(iprot, field.type); + } + iprot.readFieldEnd(); + } + iprot.readStructEnd(); + + // check for required fields of primitive type, which can't be checked in the validate method + validate(); + } + + public void write(org.apache.thrift.protocol.TProtocol oprot) throws org.apache.thrift.TException { + oprot.writeStructBegin(STRUCT_DESC); + + if (this.isSetSuccess()) { + oprot.writeFieldBegin(SUCCESS_FIELD_DESC); + { + oprot.writeListBegin(new org.apache.thrift.protocol.TList(org.apache.thrift.protocol.TType.STRUCT, this.success.size())); + for (CfSplit _iter176 : this.success) + { + _iter176.write(oprot); + } + oprot.writeListEnd(); + } + oprot.writeFieldEnd(); + } else if (this.isSetIre()) { + oprot.writeFieldBegin(IRE_FIELD_DESC); + this.ire.write(oprot); + oprot.writeFieldEnd(); + } + oprot.writeFieldStop(); + oprot.writeStructEnd(); + } + + @Override + public String toString() { + StringBuilder sb = new StringBuilder("describe_splits_ex_result("); + boolean first = true; + + sb.append("success:"); + if (this.success == null) { + sb.append("null"); + } else { + sb.append(this.success); + } + first = false; + if (!first) sb.append(", "); + sb.append("ire:"); + if (this.ire == null) { + sb.append("null"); + } else { + sb.append(this.ire); + } + first = false; + sb.append(")"); + return sb.toString(); + } + + public void validate() throws org.apache.thrift.TException { + // check for required fields + } + + private void writeObject(java.io.ObjectOutputStream out) throws java.io.IOException { + try { + write(new org.apache.thrift.protocol.TCompactProtocol(new org.apache.thrift.transport.TIOStreamTransport(out))); + } catch (org.apache.thrift.TException te) { + throw new java.io.IOException(te); + } + } + + private void readObject(java.io.ObjectInputStream in) throws java.io.IOException, ClassNotFoundException { + try { + read(new org.apache.thrift.protocol.TCompactProtocol(new org.apache.thrift.transport.TIOStreamTransport(in))); + } catch (org.apache.thrift.TException te) { + throw new java.io.IOException(te); + } + } + + } + public static class system_add_column_family_args implements org.apache.thrift.TBase, java.io.Serializable, Cloneable { private static final org.apache.thrift.protocol.TStruct STRUCT_DESC = new org.apache.thrift.protocol.TStruct("system_add_column_family_args"); @@ -34787,13 +35918,13 @@ public class Cassandra { case 2: // VALUES if (field.type == org.apache.thrift.protocol.TType.LIST) { { - org.apache.thrift.protocol.TList _list173 = iprot.readListBegin(); - this.values = new ArrayList(_list173.size); - for (int _i174 = 0; _i174 < _list173.size; ++_i174) + org.apache.thrift.protocol.TList _list177 = iprot.readListBegin(); + this.values = new ArrayList(_list177.size); + for (int _i178 = 0; _i178 < _list177.size; ++_i178) { - ByteBuffer _elem175; // required - _elem175 = iprot.readBinary(); - this.values.add(_elem175); + ByteBuffer _elem179; // required + _elem179 = iprot.readBinary(); + this.values.add(_elem179); } iprot.readListEnd(); } @@ -34826,9 +35957,9 @@ public class Cassandra { oprot.writeFieldBegin(VALUES_FIELD_DESC); { oprot.writeListBegin(new org.apache.thrift.protocol.TList(org.apache.thrift.protocol.TType.STRING, this.values.size())); - for (ByteBuffer _iter176 : this.values) + for (ByteBuffer _iter180 : this.values) { - oprot.writeBinary(_iter176); + oprot.writeBinary(_iter180); } oprot.writeListEnd(); } @@ -34876,6 +36007,8 @@ public class Cassandra { private void readObject(java.io.ObjectInputStream in) throws java.io.IOException, ClassNotFoundException { try { + // it doesn't seem like you should have to do this, but java serialization is wacky, and doesn't call the default constructor. + __isset_bit_vector = new BitSet(1); read(new org.apache.thrift.protocol.TCompactProtocol(new org.apache.thrift.transport.TIOStreamTransport(in))); } catch (org.apache.thrift.TException te) { throw new java.io.IOException(te); diff --git a/interface/thrift/gen-java/org/apache/cassandra/thrift/CfSplit.java b/interface/thrift/gen-java/org/apache/cassandra/thrift/CfSplit.java new file mode 100644 index 0000000000..2519f9f4b1 --- /dev/null +++ b/interface/thrift/gen-java/org/apache/cassandra/thrift/CfSplit.java @@ -0,0 +1,549 @@ +/** + * Autogenerated by Thrift Compiler (0.7.0) + * + * DO NOT EDIT UNLESS YOU ARE SURE THAT YOU KNOW WHAT YOU ARE DOING + */ +package org.apache.cassandra.thrift; +/* + * + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + * + */ + + +import org.apache.commons.lang.builder.HashCodeBuilder; +import java.util.List; +import java.util.ArrayList; +import java.util.Map; +import java.util.HashMap; +import java.util.EnumMap; +import java.util.Set; +import java.util.HashSet; +import java.util.EnumSet; +import java.util.Collections; +import java.util.BitSet; +import java.nio.ByteBuffer; +import java.util.Arrays; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +/** + * Represents input splits used by hadoop ColumnFamilyRecordReaders + */ +public class CfSplit implements org.apache.thrift.TBase, java.io.Serializable, Cloneable { + private static final org.apache.thrift.protocol.TStruct STRUCT_DESC = new org.apache.thrift.protocol.TStruct("CfSplit"); + + private static final org.apache.thrift.protocol.TField START_TOKEN_FIELD_DESC = new org.apache.thrift.protocol.TField("start_token", org.apache.thrift.protocol.TType.STRING, (short)1); + private static final org.apache.thrift.protocol.TField END_TOKEN_FIELD_DESC = new org.apache.thrift.protocol.TField("end_token", org.apache.thrift.protocol.TType.STRING, (short)2); + private static final org.apache.thrift.protocol.TField ROW_COUNT_FIELD_DESC = new org.apache.thrift.protocol.TField("row_count", org.apache.thrift.protocol.TType.I64, (short)3); + + public String start_token; // required + public String end_token; // required + public long row_count; // required + + /** The set of fields this struct contains, along with convenience methods for finding and manipulating them. */ + public enum _Fields implements org.apache.thrift.TFieldIdEnum { + START_TOKEN((short)1, "start_token"), + END_TOKEN((short)2, "end_token"), + ROW_COUNT((short)3, "row_count"); + + private static final Map byName = new HashMap(); + + static { + for (_Fields field : EnumSet.allOf(_Fields.class)) { + byName.put(field.getFieldName(), field); + } + } + + /** + * Find the _Fields constant that matches fieldId, or null if its not found. + */ + public static _Fields findByThriftId(int fieldId) { + switch(fieldId) { + case 1: // START_TOKEN + return START_TOKEN; + case 2: // END_TOKEN + return END_TOKEN; + case 3: // ROW_COUNT + return ROW_COUNT; + default: + return null; + } + } + + /** + * Find the _Fields constant that matches fieldId, throwing an exception + * if it is not found. + */ + public static _Fields findByThriftIdOrThrow(int fieldId) { + _Fields fields = findByThriftId(fieldId); + if (fields == null) throw new IllegalArgumentException("Field " + fieldId + " doesn't exist!"); + return fields; + } + + /** + * Find the _Fields constant that matches name, or null if its not found. + */ + public static _Fields findByName(String name) { + return byName.get(name); + } + + private final short _thriftId; + private final String _fieldName; + + _Fields(short thriftId, String fieldName) { + _thriftId = thriftId; + _fieldName = fieldName; + } + + public short getThriftFieldId() { + return _thriftId; + } + + public String getFieldName() { + return _fieldName; + } + } + + // isset id assignments + private static final int __ROW_COUNT_ISSET_ID = 0; + private BitSet __isset_bit_vector = new BitSet(1); + + public static final Map<_Fields, org.apache.thrift.meta_data.FieldMetaData> metaDataMap; + static { + Map<_Fields, org.apache.thrift.meta_data.FieldMetaData> tmpMap = new EnumMap<_Fields, org.apache.thrift.meta_data.FieldMetaData>(_Fields.class); + tmpMap.put(_Fields.START_TOKEN, new org.apache.thrift.meta_data.FieldMetaData("start_token", org.apache.thrift.TFieldRequirementType.REQUIRED, + new org.apache.thrift.meta_data.FieldValueMetaData(org.apache.thrift.protocol.TType.STRING))); + tmpMap.put(_Fields.END_TOKEN, new org.apache.thrift.meta_data.FieldMetaData("end_token", org.apache.thrift.TFieldRequirementType.REQUIRED, + new org.apache.thrift.meta_data.FieldValueMetaData(org.apache.thrift.protocol.TType.STRING))); + tmpMap.put(_Fields.ROW_COUNT, new org.apache.thrift.meta_data.FieldMetaData("row_count", org.apache.thrift.TFieldRequirementType.REQUIRED, + new org.apache.thrift.meta_data.FieldValueMetaData(org.apache.thrift.protocol.TType.I64))); + metaDataMap = Collections.unmodifiableMap(tmpMap); + org.apache.thrift.meta_data.FieldMetaData.addStructMetaDataMap(CfSplit.class, metaDataMap); + } + + public CfSplit() { + } + + public CfSplit( + String start_token, + String end_token, + long row_count) + { + this(); + this.start_token = start_token; + this.end_token = end_token; + this.row_count = row_count; + setRow_countIsSet(true); + } + + /** + * Performs a deep copy on other. + */ + public CfSplit(CfSplit other) { + __isset_bit_vector.clear(); + __isset_bit_vector.or(other.__isset_bit_vector); + if (other.isSetStart_token()) { + this.start_token = other.start_token; + } + if (other.isSetEnd_token()) { + this.end_token = other.end_token; + } + this.row_count = other.row_count; + } + + public CfSplit deepCopy() { + return new CfSplit(this); + } + + @Override + public void clear() { + this.start_token = null; + this.end_token = null; + setRow_countIsSet(false); + this.row_count = 0; + } + + public String getStart_token() { + return this.start_token; + } + + public CfSplit setStart_token(String start_token) { + this.start_token = start_token; + return this; + } + + public void unsetStart_token() { + this.start_token = null; + } + + /** Returns true if field start_token is set (has been assigned a value) and false otherwise */ + public boolean isSetStart_token() { + return this.start_token != null; + } + + public void setStart_tokenIsSet(boolean value) { + if (!value) { + this.start_token = null; + } + } + + public String getEnd_token() { + return this.end_token; + } + + public CfSplit setEnd_token(String end_token) { + this.end_token = end_token; + return this; + } + + public void unsetEnd_token() { + this.end_token = null; + } + + /** Returns true if field end_token is set (has been assigned a value) and false otherwise */ + public boolean isSetEnd_token() { + return this.end_token != null; + } + + public void setEnd_tokenIsSet(boolean value) { + if (!value) { + this.end_token = null; + } + } + + public long getRow_count() { + return this.row_count; + } + + public CfSplit setRow_count(long row_count) { + this.row_count = row_count; + setRow_countIsSet(true); + return this; + } + + public void unsetRow_count() { + __isset_bit_vector.clear(__ROW_COUNT_ISSET_ID); + } + + /** Returns true if field row_count is set (has been assigned a value) and false otherwise */ + public boolean isSetRow_count() { + return __isset_bit_vector.get(__ROW_COUNT_ISSET_ID); + } + + public void setRow_countIsSet(boolean value) { + __isset_bit_vector.set(__ROW_COUNT_ISSET_ID, value); + } + + public void setFieldValue(_Fields field, Object value) { + switch (field) { + case START_TOKEN: + if (value == null) { + unsetStart_token(); + } else { + setStart_token((String)value); + } + break; + + case END_TOKEN: + if (value == null) { + unsetEnd_token(); + } else { + setEnd_token((String)value); + } + break; + + case ROW_COUNT: + if (value == null) { + unsetRow_count(); + } else { + setRow_count((Long)value); + } + break; + + } + } + + public Object getFieldValue(_Fields field) { + switch (field) { + case START_TOKEN: + return getStart_token(); + + case END_TOKEN: + return getEnd_token(); + + case ROW_COUNT: + return Long.valueOf(getRow_count()); + + } + throw new IllegalStateException(); + } + + /** Returns true if field corresponding to fieldID is set (has been assigned a value) and false otherwise */ + public boolean isSet(_Fields field) { + if (field == null) { + throw new IllegalArgumentException(); + } + + switch (field) { + case START_TOKEN: + return isSetStart_token(); + case END_TOKEN: + return isSetEnd_token(); + case ROW_COUNT: + return isSetRow_count(); + } + throw new IllegalStateException(); + } + + @Override + public boolean equals(Object that) { + if (that == null) + return false; + if (that instanceof CfSplit) + return this.equals((CfSplit)that); + return false; + } + + public boolean equals(CfSplit that) { + if (that == null) + return false; + + boolean this_present_start_token = true && this.isSetStart_token(); + boolean that_present_start_token = true && that.isSetStart_token(); + if (this_present_start_token || that_present_start_token) { + if (!(this_present_start_token && that_present_start_token)) + return false; + if (!this.start_token.equals(that.start_token)) + return false; + } + + boolean this_present_end_token = true && this.isSetEnd_token(); + boolean that_present_end_token = true && that.isSetEnd_token(); + if (this_present_end_token || that_present_end_token) { + if (!(this_present_end_token && that_present_end_token)) + return false; + if (!this.end_token.equals(that.end_token)) + return false; + } + + boolean this_present_row_count = true; + boolean that_present_row_count = true; + if (this_present_row_count || that_present_row_count) { + if (!(this_present_row_count && that_present_row_count)) + return false; + if (this.row_count != that.row_count) + return false; + } + + return true; + } + + @Override + public int hashCode() { + HashCodeBuilder builder = new HashCodeBuilder(); + + boolean present_start_token = true && (isSetStart_token()); + builder.append(present_start_token); + if (present_start_token) + builder.append(start_token); + + boolean present_end_token = true && (isSetEnd_token()); + builder.append(present_end_token); + if (present_end_token) + builder.append(end_token); + + boolean present_row_count = true; + builder.append(present_row_count); + if (present_row_count) + builder.append(row_count); + + return builder.toHashCode(); + } + + public int compareTo(CfSplit other) { + if (!getClass().equals(other.getClass())) { + return getClass().getName().compareTo(other.getClass().getName()); + } + + int lastComparison = 0; + CfSplit typedOther = (CfSplit)other; + + lastComparison = Boolean.valueOf(isSetStart_token()).compareTo(typedOther.isSetStart_token()); + if (lastComparison != 0) { + return lastComparison; + } + if (isSetStart_token()) { + lastComparison = org.apache.thrift.TBaseHelper.compareTo(this.start_token, typedOther.start_token); + if (lastComparison != 0) { + return lastComparison; + } + } + lastComparison = Boolean.valueOf(isSetEnd_token()).compareTo(typedOther.isSetEnd_token()); + if (lastComparison != 0) { + return lastComparison; + } + if (isSetEnd_token()) { + lastComparison = org.apache.thrift.TBaseHelper.compareTo(this.end_token, typedOther.end_token); + if (lastComparison != 0) { + return lastComparison; + } + } + lastComparison = Boolean.valueOf(isSetRow_count()).compareTo(typedOther.isSetRow_count()); + if (lastComparison != 0) { + return lastComparison; + } + if (isSetRow_count()) { + lastComparison = org.apache.thrift.TBaseHelper.compareTo(this.row_count, typedOther.row_count); + if (lastComparison != 0) { + return lastComparison; + } + } + return 0; + } + + public _Fields fieldForId(int fieldId) { + return _Fields.findByThriftId(fieldId); + } + + public void read(org.apache.thrift.protocol.TProtocol iprot) throws org.apache.thrift.TException { + org.apache.thrift.protocol.TField field; + iprot.readStructBegin(); + while (true) + { + field = iprot.readFieldBegin(); + if (field.type == org.apache.thrift.protocol.TType.STOP) { + break; + } + switch (field.id) { + case 1: // START_TOKEN + if (field.type == org.apache.thrift.protocol.TType.STRING) { + this.start_token = iprot.readString(); + } else { + org.apache.thrift.protocol.TProtocolUtil.skip(iprot, field.type); + } + break; + case 2: // END_TOKEN + if (field.type == org.apache.thrift.protocol.TType.STRING) { + this.end_token = iprot.readString(); + } else { + org.apache.thrift.protocol.TProtocolUtil.skip(iprot, field.type); + } + break; + case 3: // ROW_COUNT + if (field.type == org.apache.thrift.protocol.TType.I64) { + this.row_count = iprot.readI64(); + setRow_countIsSet(true); + } else { + org.apache.thrift.protocol.TProtocolUtil.skip(iprot, field.type); + } + break; + default: + org.apache.thrift.protocol.TProtocolUtil.skip(iprot, field.type); + } + iprot.readFieldEnd(); + } + iprot.readStructEnd(); + + // check for required fields of primitive type, which can't be checked in the validate method + if (!isSetRow_count()) { + throw new org.apache.thrift.protocol.TProtocolException("Required field 'row_count' was not found in serialized data! Struct: " + toString()); + } + validate(); + } + + public void write(org.apache.thrift.protocol.TProtocol oprot) throws org.apache.thrift.TException { + validate(); + + oprot.writeStructBegin(STRUCT_DESC); + if (this.start_token != null) { + oprot.writeFieldBegin(START_TOKEN_FIELD_DESC); + oprot.writeString(this.start_token); + oprot.writeFieldEnd(); + } + if (this.end_token != null) { + oprot.writeFieldBegin(END_TOKEN_FIELD_DESC); + oprot.writeString(this.end_token); + oprot.writeFieldEnd(); + } + oprot.writeFieldBegin(ROW_COUNT_FIELD_DESC); + oprot.writeI64(this.row_count); + oprot.writeFieldEnd(); + oprot.writeFieldStop(); + oprot.writeStructEnd(); + } + + @Override + public String toString() { + StringBuilder sb = new StringBuilder("CfSplit("); + boolean first = true; + + sb.append("start_token:"); + if (this.start_token == null) { + sb.append("null"); + } else { + sb.append(this.start_token); + } + first = false; + if (!first) sb.append(", "); + sb.append("end_token:"); + if (this.end_token == null) { + sb.append("null"); + } else { + sb.append(this.end_token); + } + first = false; + if (!first) sb.append(", "); + sb.append("row_count:"); + sb.append(this.row_count); + first = false; + sb.append(")"); + return sb.toString(); + } + + public void validate() throws org.apache.thrift.TException { + // check for required fields + if (start_token == null) { + throw new org.apache.thrift.protocol.TProtocolException("Required field 'start_token' was not present! Struct: " + toString()); + } + if (end_token == null) { + throw new org.apache.thrift.protocol.TProtocolException("Required field 'end_token' was not present! Struct: " + toString()); + } + // alas, we cannot check 'row_count' because it's a primitive and you chose the non-beans generator. + } + + private void writeObject(java.io.ObjectOutputStream out) throws java.io.IOException { + try { + write(new org.apache.thrift.protocol.TCompactProtocol(new org.apache.thrift.transport.TIOStreamTransport(out))); + } catch (org.apache.thrift.TException te) { + throw new java.io.IOException(te); + } + } + + private void readObject(java.io.ObjectInputStream in) throws java.io.IOException, ClassNotFoundException { + try { + // it doesn't seem like you should have to do this, but java serialization is wacky, and doesn't call the default constructor. + __isset_bit_vector = new BitSet(1); + read(new org.apache.thrift.protocol.TCompactProtocol(new org.apache.thrift.transport.TIOStreamTransport(in))); + } catch (org.apache.thrift.TException te) { + throw new java.io.IOException(te); + } + } + +} + diff --git a/interface/thrift/gen-java/org/apache/cassandra/thrift/Constants.java b/interface/thrift/gen-java/org/apache/cassandra/thrift/Constants.java index 7e183c79fc..9d0701fc1e 100644 --- a/interface/thrift/gen-java/org/apache/cassandra/thrift/Constants.java +++ b/interface/thrift/gen-java/org/apache/cassandra/thrift/Constants.java @@ -44,6 +44,6 @@ import org.slf4j.LoggerFactory; public class Constants { - public static final String VERSION = "19.32.0"; + public static final String VERSION = "19.33.0"; } diff --git a/src/java/org/apache/cassandra/hadoop/ColumnFamilyInputFormat.java b/src/java/org/apache/cassandra/hadoop/ColumnFamilyInputFormat.java index cb79b01302..c4c6570024 100644 --- a/src/java/org/apache/cassandra/hadoop/ColumnFamilyInputFormat.java +++ b/src/java/org/apache/cassandra/hadoop/ColumnFamilyInputFormat.java @@ -35,6 +35,9 @@ import java.util.concurrent.Executors; import java.util.concurrent.Future; import com.google.common.collect.ImmutableList; +import org.apache.commons.lang.StringUtils; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; import org.apache.cassandra.db.IColumn; import org.apache.cassandra.dht.IPartitioner; @@ -44,18 +47,11 @@ import org.apache.cassandra.thrift.Cassandra; import org.apache.cassandra.thrift.InvalidRequestException; import org.apache.cassandra.thrift.KeyRange; import org.apache.cassandra.thrift.TokenRange; -import org.apache.commons.lang.StringUtils; import org.apache.hadoop.conf.Configuration; -import org.apache.hadoop.mapred.*; -import org.apache.hadoop.mapreduce.InputFormat; -import org.apache.hadoop.mapreduce.InputSplit; -import org.apache.hadoop.mapreduce.JobContext; -import org.apache.hadoop.mapreduce.RecordReader; -import org.apache.hadoop.mapreduce.TaskAttemptContext; -import org.apache.hadoop.mapreduce.TaskAttemptID; +import org.apache.hadoop.mapred.JobConf; +import org.apache.hadoop.mapred.Reporter; +import org.apache.hadoop.mapreduce.*; import org.apache.thrift.TException; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; /** * Hadoop InputFormat allowing map/reduce against Cassandra rows within one ColumnFamily. @@ -208,7 +204,7 @@ public class ColumnFamilyInputFormat extends InputFormat call() throws Exception { ArrayList splits = new ArrayList(); - List tokens = getSubSplits(keyspace, cfName, range, conf); + List subSplits = getSubSplits(keyspace, cfName, range, conf); assert range.rpc_endpoints.size() == range.endpoints.size() : "rpc_endpoints size must match endpoints size"; // turn the sub-ranges into InputSplits String[] endpoints = range.endpoints.toArray(new String[range.endpoints.size()]); @@ -223,15 +219,21 @@ public class ColumnFamilyInputFormat extends InputFormat range = new Range(left, right, partitioner); List> ranges = range.isWrapAround() ? range.unwrap() : ImmutableList.of(range); for (Range subrange : ranges) { - ColumnFamilySplit split = new ColumnFamilySplit(factory.toString(subrange.left), factory.toString(subrange.right), endpoints); + ColumnFamilySplit split = + new ColumnFamilySplit( + factory.toString(subrange.left), + factory.toString(subrange.right), + subSplit.getRow_count(), + endpoints); + logger.debug("adding " + split); splits.add(split); } @@ -240,7 +242,7 @@ public class ColumnFamilyInputFormat extends InputFormat getSubSplits(String keyspace, String cfName, TokenRange range, Configuration conf) throws IOException + private List getSubSplits(String keyspace, String cfName, TokenRange range, Configuration conf) throws IOException { int splitsize = ConfigHelper.getInputSplitSize(conf); for (int i = 0; i < range.rpc_endpoints.size(); i++) @@ -254,7 +256,7 @@ public class ColumnFamilyInputFormat extends InputFormat> next = wideColumns.next(); lastColumn = next.right.values().iterator().next().name(); - maybeCountRow(next); + maybeIncreaseRowCounter(next); return next; } @@ -491,7 +494,7 @@ public class ColumnFamilyRecordReader extends RecordReader> next) + private void maybeIncreaseRowCounter(Pair> next) { ByteBuffer currentKey = next.left; if (!currentKey.equals(lastCountedKey)) diff --git a/src/java/org/apache/cassandra/hadoop/ColumnFamilySplit.java b/src/java/org/apache/cassandra/hadoop/ColumnFamilySplit.java index bd2e4873d7..4085c68a3d 100644 --- a/src/java/org/apache/cassandra/hadoop/ColumnFamilySplit.java +++ b/src/java/org/apache/cassandra/hadoop/ColumnFamilySplit.java @@ -33,14 +33,22 @@ public class ColumnFamilySplit extends InputSplit implements Writable, org.apach { private String startToken; private String endToken; + private long length; private String[] dataNodes; + @Deprecated public ColumnFamilySplit(String startToken, String endToken, String[] dataNodes) + { + this(startToken, endToken, Long.MAX_VALUE, dataNodes); + } + + public ColumnFamilySplit(String startToken, String endToken, long length, String[] dataNodes) { assert startToken != null; assert endToken != null; this.startToken = startToken; this.endToken = endToken; + this.length = length; this.dataNodes = dataNodes; } @@ -58,8 +66,7 @@ public class ColumnFamilySplit extends InputSplit implements Writable, org.apach public long getLength() { - // only used for sorting splits. we don't have the capability, yet. - return Long.MAX_VALUE; + return length; } public String[] getLocations() @@ -76,7 +83,7 @@ public class ColumnFamilySplit extends InputSplit implements Writable, org.apach { out.writeUTF(startToken); out.writeUTF(endToken); - + out.writeLong(length); out.writeInt(dataNodes.length); for (String endpoint : dataNodes) { @@ -88,6 +95,7 @@ public class ColumnFamilySplit extends InputSplit implements Writable, org.apach { startToken = in.readUTF(); endToken = in.readUTF(); + length = in.readLong(); int numOfEndpoints = in.readInt(); dataNodes = new String[numOfEndpoints]; diff --git a/src/java/org/apache/cassandra/service/StorageService.java b/src/java/org/apache/cassandra/service/StorageService.java index b1eaa1ecf5..80c3f46902 100644 --- a/src/java/org/apache/cassandra/service/StorageService.java +++ b/src/java/org/apache/cassandra/service/StorageService.java @@ -36,6 +36,7 @@ import com.google.common.base.Supplier; import com.google.common.collect.*; import org.apache.cassandra.metrics.ClientRequestMetrics; + import org.apache.log4j.Level; import org.apache.commons.lang.StringUtils; import org.slf4j.Logger; @@ -2184,28 +2185,50 @@ public class StorageService implements IEndpointStateChangeSubscriber, StorageSe } /** - * @return list of Tokens (_not_ keys!) breaking up the data this node is responsible for into pieces of roughly keysPerSplit + * @return list of Token ranges (_not_ keys!) together with estimated key count, + * breaking up the data this node is responsible for into pieces of roughly keysPerSplit */ - public List getSplits(String table, String cfName, Range range, int keysPerSplit) + public List, Long>> getSplits(String table, String cfName, Range range, int keysPerSplit) { - List tokens = new ArrayList(); - // we use the actual Range token for the first and last brackets of the splits to ensure correctness - tokens.add(range.left); - Table t = Table.open(table); ColumnFamilyStore cfs = t.getColumnFamilyStore(cfName); List keys = keySamples(Collections.singleton(cfs), range); - int splits = keys.size() * DatabaseDescriptor.getIndexInterval() / keysPerSplit; - if (keys.size() >= splits) + final long totalRowCountEstimate = (keys.size() + 1) * DatabaseDescriptor.getIndexInterval(); + + // splitCount should be much smaller than number of key samples, to avoid huge sampling error + final int minSamplesPerSplit = 4; + final int maxSplitCount = keys.size() / minSamplesPerSplit + 1; + final int splitCount = Math.max(1, Math.min(maxSplitCount, (int)(totalRowCountEstimate / keysPerSplit))); + + List tokens = keysToTokens(range, keys); + return getSplits(tokens, splitCount); + } + + private List, Long>> getSplits(List tokens, int splitCount) + { + final double step = (double) (tokens.size() - 1) / splitCount; + int prevIndex = 0; + Token prevToken = tokens.get(0); + List, Long>> splits = Lists.newArrayListWithExpectedSize(splitCount); + for (int i = 1; i <= splitCount; i++) { - for (int i = 1; i < splits; i++) - { - int index = i * (keys.size() / splits); - tokens.add(keys.get(index).token); - } + int index = (int) Math.round(i * step); + Token token = tokens.get(index); + long rowCountEstimate = (index - prevIndex) * DatabaseDescriptor.getIndexInterval(); + splits.add(Pair.create(new Range(prevToken, token), rowCountEstimate)); + prevIndex = index; + prevToken = token; } + return splits; + } + private List keysToTokens(Range range, List keys) + { + List tokens = Lists.newArrayListWithExpectedSize(keys.size() + 2); + tokens.add(range.left); + for (DecoratedKey key : keys) + tokens.add(key.token); tokens.add(range.right); return tokens; } diff --git a/src/java/org/apache/cassandra/thrift/CassandraServer.java b/src/java/org/apache/cassandra/thrift/CassandraServer.java index ad416f32ed..3bf155e1b8 100644 --- a/src/java/org/apache/cassandra/thrift/CassandraServer.java +++ b/src/java/org/apache/cassandra/thrift/CassandraServer.java @@ -30,6 +30,8 @@ import java.util.zip.Inflater; import com.google.common.base.Predicates; import com.google.common.collect.Maps; +import org.apache.cassandra.hadoop.ColumnFamilySplit; +import org.apache.cassandra.utils.Pair; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -882,18 +884,33 @@ public class CassandraServer implements Cassandra.Iface return DatabaseDescriptor.getEndpointSnitch().getClass().getName(); } + @Deprecated public List describe_splits(String cfName, String start_token, String end_token, int keys_per_split) throws TException, InvalidRequestException + { + List splits = describe_splits_ex(cfName, start_token, end_token, keys_per_split); + List result = new ArrayList(splits.size() + 1); + + result.add(splits.get(0).getStart_token()); + for (CfSplit cfSplit : splits) + result.add(cfSplit.getEnd_token()); + + return result; + } + + @Override + public List describe_splits_ex(String cfName, String start_token, String end_token, int keys_per_split) + throws InvalidRequestException, TException { // TODO: add keyspace authorization call post CASSANDRA-1425 Token.TokenFactory tf = StorageService.getPartitioner().getTokenFactory(); - List tokens = StorageService.instance.getSplits(state().getKeyspace(), cfName, new Range(tf.fromString(start_token), tf.fromString(end_token)), keys_per_split); - List splits = new ArrayList(tokens.size()); - for (Token token : tokens) - { - splits.add(tf.toString(token)); - } - return splits; + Range tr = new Range(tf.fromString(start_token), tf.fromString(end_token)); + List, Long>> splits = + StorageService.instance.getSplits(state().getKeyspace(), cfName, tr, keys_per_split); + List result = new ArrayList(splits.size()); + for (Pair, Long> split : splits) + result.add(new CfSplit(split.left.left.toString(), split.left.right.toString(), split.right)); + return result; } public void login(AuthenticationRequest auth_request) throws AuthenticationException, AuthorizationException, TException