From fc468f682b58f8d37e8d81d320f53c22a0a20008 Mon Sep 17 00:00:00 2001 From: "A.J. Beamon" Date: Fri, 26 May 2017 12:33:33 -0700 Subject: [PATCH] Merge branch 'release-5.0' into bindings-tuple-improvements # Conflicts: # bindings/java/src-completable/main/com/apple/apple/foundationdbdb/tuple/Tuple.java --- bindings/bindingtester/known_testers.py | 10 +- .../bindingtester/spec/bindingApiTester.txt | 34 ++ bindings/bindingtester/tests/api.py | 50 +- bindings/bindingtester/tests/directory.py | 2 +- bindings/bindingtester/tests/directory_hca.py | 2 +- bindings/bindingtester/tests/test_util.py | 66 ++- bindings/flow/DirectoryLayer.actor.cpp | 2 +- bindings/flow/DirectoryLayer.h | 2 +- bindings/flow/HighContentionAllocator.h | 2 +- bindings/flow/Subspace.h | 12 + bindings/flow/Tuple.cpp | 428 +++++++++++++++++- bindings/flow/Tuple.h | 53 ++- .../flow/tester/DirectoryTester.actor.cpp | 10 +- bindings/flow/tester/Tester.actor.cpp | 188 +++++++- bindings/go/src/_stacktester/stacktester.go | 103 ++++- bindings/go/src/fdb/tuple/tuple.go | 221 +++++++-- .../apple/cie/foundationdb/FDBDatabase.java | 2 +- .../tuple/IterableComparator.java | 107 +++++ .../apple/cie/foundationdb/tuple/Tuple.java | 311 ++++++++++++- .../cie/foundationdb/tuple/TupleUtil.java | 354 +++++++++++++-- .../foundationdb/test/AsyncStackTester.java | 72 ++- .../cie/foundationdb/test/StackOperation.java | 5 + .../cie/foundationdb/test/StackTester.java | 42 +- .../cie/foundationdb/test/StackUtils.java | 9 + .../tuple/IterableComparator.java | 107 +++++ .../apple/cie/foundationdb/tuple/Tuple.java | 309 ++++++++++++- .../cie/foundationdb/tuple/TupleUtil.java | 355 +++++++++++++-- .../foundationdb/test/AsyncStackTester.java | 72 ++- .../cie/foundationdb/test/StackOperation.java | 5 + .../cie/foundationdb/test/StackTester.java | 44 +- .../cie/foundationdb/test/StackUtils.java | 9 + bindings/nodejs/lib/fdb.js | 4 +- bindings/nodejs/lib/tuple.js | 253 ++++++++++- bindings/nodejs/tests/tester.js | 38 ++ bindings/nodejs/tests/tuple_test.js | 40 +- bindings/python/fdb/tuple.py | 248 +++++++++- bindings/python/tests/tester.py | 28 ++ bindings/python/tests/tuple_tests.py | 131 ++++-- bindings/ruby/lib/fdbtuple.rb | 204 ++++++++- bindings/ruby/tests/tester.rb | 21 + flow/error_definitions.h | 1 + 41 files changed, 3680 insertions(+), 276 deletions(-) create mode 100644 bindings/java/src-completable/main/com/apple/cie/foundationdb/tuple/IterableComparator.java create mode 100644 bindings/java/src/main/com/apple/cie/foundationdb/tuple/IterableComparator.java diff --git a/bindings/bindingtester/known_testers.py b/bindings/bindingtester/known_testers.py index 56cfd4669e..09d0a9b786 100644 --- a/bindings/bindingtester/known_testers.py +++ b/bindings/bindingtester/known_testers.py @@ -56,13 +56,13 @@ _java_completable_cmd = 'java -ea -cp %s:%s com.apple.cie.foundationdb.test.' % testers = { 'python' : Tester('python', 'python ' + _absolute_path('python/tests/tester.py'), 2040, 23, MAX_API_VERSION), 'python3' : Tester('python3', 'python3 ' + _absolute_path('python/tests/tester.py'), 2040, 23, MAX_API_VERSION), - 'node' : Tester('node', _absolute_path('nodejs/tests/tester.js'), 53, 23, MAX_API_VERSION), + 'node' : Tester('node', _absolute_path('nodejs/tests/tester.js'), 53, 500, MAX_API_VERSION), 'streamline' : Tester('streamline', _absolute_path('nodejs/tests/streamline_tester._js'), 53, 23, MAX_API_VERSION), 'ruby' : Tester('ruby', _absolute_path('ruby/tests/tester.rb'), 64, 23, MAX_API_VERSION), - 'java' : Tester('java', _java_cmd + 'StackTester', 63, 500, MAX_API_VERSION), - 'java_async' : Tester('java', _java_cmd + 'AsyncStackTester', 63, 500, MAX_API_VERSION), - 'java_completable' : Tester('java', _java_completable_cmd + 'StackTester', 63, 500, MAX_API_VERSION), - 'java_completable_async' : Tester('java', _java_completable_cmd + 'AsyncStackTester', 63, 500, MAX_API_VERSION), + 'java' : Tester('java', _java_cmd + 'StackTester', 2040, 500, MAX_API_VERSION), + 'java_async' : Tester('java', _java_cmd + 'AsyncStackTester', 2040, 500, MAX_API_VERSION), + 'java_completable' : Tester('java', _java_completable_cmd + 'StackTester', 2040, 500, MAX_API_VERSION), + 'java_completable_async' : Tester('java', _java_completable_cmd + 'AsyncStackTester', 2040, 500, MAX_API_VERSION), 'go' : Tester('go', _absolute_path('go/bin/_stacktester'), 63, 200, MAX_API_VERSION), 'flow' : Tester('flow', _absolute_path('flow/bin/fdb_flow_tester'), 63, 200, MAX_API_VERSION), } diff --git a/bindings/bindingtester/spec/bindingApiTester.txt b/bindings/bindingtester/spec/bindingApiTester.txt index 6b2c05bffe..3e96e8ecab 100644 --- a/bindings/bindingtester/spec/bindingApiTester.txt +++ b/bindings/bindingtester/spec/bindingApiTester.txt @@ -306,6 +306,40 @@ TUPLE_RANGE structure) to the tuple range method. Pushes the begin and end elements of the returned range onto the stack. +TUPLE_SORT + + Pops the top item off of the stack as N. Pops the next N items off of the + stack as packed tuples (i.e., byte strings), unpacks them, sorts the tuples, + repacks them into byte strings, and then pushes these packed tuples onto + the stack so that the final top of the stack now has the greatest + element. If the binding has some kind of tuple comparison function, it should + use that to sort. Otherwise, it should sort them lexicographically by + their byte representation. The choice of function should not affect final sort order. + +ENCODE_FLOAT + + Pops the top item off of the stack. This will be a byte-string of length 4 + containing the IEEE 754 encoding of a float in big-endian order. + This is then converted into a float and pushed onto the stack. + +ENCODE_DOUBLE + + Pops the top item off of the stack. This will be a byte-string of length 8 + containing the IEEE 754 encoding of a double in big-endian order. + This is then converted into a double and pushed onto the stack. + +DECODE_FLOAT + + Pops the top item off of the stack. This will be a single-precision float. + This is converted into a (4 byte) byte-string of its IEEE 754 representation + in big-endian order, and pushed onto the stack. + +DECODE_DOUBLE + + Pops the top item off of the stack. This will be a double-precision float. + This is converted into a (8 byte) byte-string its IEEE 754 representation + in big-endian order, and pushed onto the stack. + Thread Operations ----------------- diff --git a/bindings/bindingtester/tests/api.py b/bindings/bindingtester/tests/api.py index 1b1e92d3d0..8b832d139e 100644 --- a/bindings/bindingtester/tests/api.py +++ b/bindings/bindingtester/tests/api.py @@ -18,9 +18,12 @@ # limitations under the License. # +import ctypes import random +import struct import fdb +import fdb.tuple from bindingtester import FDB_API_VERSION from bindingtester.tests import Test, Instruction, InstructionSet, ResultSpecification @@ -51,7 +54,7 @@ class ApiTest(Test): self.generated_keys = [] self.outstanding_ops = [] - self.random = test_util.RandomGenerator(args.max_int_bits) + self.random = test_util.RandomGenerator(args.max_int_bits, args.api_version) def add_stack_items(self, num): self.stack_size += num @@ -144,7 +147,7 @@ class ApiTest(Test): mutations += ['VERSIONSTAMP'] versions = ['GET_READ_VERSION', 'SET_READ_VERSION', 'GET_COMMITTED_VERSION'] snapshot_versions = ['GET_READ_VERSION_SNAPSHOT'] - tuples = ['TUPLE_PACK', 'TUPLE_UNPACK', 'TUPLE_RANGE', 'SUB'] + tuples = ['TUPLE_PACK', 'TUPLE_UNPACK', 'TUPLE_RANGE', 'TUPLE_SORT', 'SUB', 'ENCODE_FLOAT', 'ENCODE_DOUBLE', 'DECODE_DOUBLE', 'DECODE_FLOAT'] resets = ['ON_ERROR', 'RESET', 'CANCEL'] read_conflicts = ['READ_CONFLICT_RANGE', 'READ_CONFLICT_KEY'] write_conflicts = ['WRITE_CONFLICT_RANGE', 'WRITE_CONFLICT_KEY', 'DISABLE_WRITE_CONFLICT'] @@ -179,6 +182,7 @@ class ApiTest(Test): for i in range(args.num_ops): op = random.choice(op_choices) index = len(instructions) + read_performed = False #print 'Adding instruction %s at %d' % (op, index) @@ -218,6 +222,7 @@ class ApiTest(Test): instructions.append(op) self.add_strings(1) self.can_set_version = False + read_performed = True elif op == 'GET_KEY' or op == 'GET_KEY_SNAPSHOT' or op == 'GET_KEY_DATABASE': if op.endswith('_DATABASE') or self.can_use_key_selectors: @@ -230,6 +235,7 @@ class ApiTest(Test): #Don't add key here because we may be outside of our prefix self.add_strings(1) self.can_set_version = False + read_performed = True elif op == 'GET_RANGE' or op == 'GET_RANGE_SNAPSHOT' or op == 'GET_RANGE_DATABASE': self.ensure_key(instructions, 2) @@ -245,6 +251,7 @@ class ApiTest(Test): self.add_stack_items(1) self.can_set_version = False + read_performed = True elif op == 'GET_RANGE_STARTS_WITH' or op == 'GET_RANGE_STARTS_WITH_SNAPSHOT' or op == 'GET_RANGE_STARTS_WITH_DATABASE': #TODO: not tested well @@ -260,6 +267,7 @@ class ApiTest(Test): self.add_stack_items(1) self.can_set_version = False + read_performed = True elif op == 'GET_RANGE_SELECTOR' or op == 'GET_RANGE_SELECTOR_SNAPSHOT' or op == 'GET_RANGE_SELECTOR_DATABASE': if op.endswith('_DATABASE') or self.can_use_key_selectors: @@ -279,6 +287,7 @@ class ApiTest(Test): self.add_stack_items(1) self.can_set_version = False + read_performed = True elif op == 'GET_READ_VERSION' or op == 'GET_READ_VERSION_SNAPSHOT': instructions.append(op) @@ -409,6 +418,15 @@ class ApiTest(Test): instructions.append(op) self.add_strings(len(tup)) + elif op == 'TUPLE_SORT': + tups = self.random.random_tuple_list(10, 30) + for tup in tups: + instructions.push_args(len(tup), *tup) + instructions.append('TUPLE_PACK') + instructions.push_args(len(tups)) + instructions.append(op) + self.add_strings(len(tups)) + #Use SUB to test if integers are correctly unpacked elif op == 'SUB': a = self.random.random_int() / 2 @@ -422,10 +440,36 @@ class ApiTest(Test): instructions.append('TUPLE_PACK') self.add_stack_items(1) + elif op == 'ENCODE_FLOAT': + f = self.random.random_float(8) + f_bytes = struct.pack('>f', f) + instructions.push_args(f_bytes) + instructions.append(op) + self.add_stack_items(1) + + elif op == 'ENCODE_DOUBLE': + d = self.random.random_float(11) + d_bytes = struct.pack('>d', d) + instructions.push_args(d_bytes) + instructions.append(op) + self.add_stack_items(1) + + elif op == 'DECODE_FLOAT': + f = self.random.random_float(8) + instructions.push_args(fdb.tuple.SingleFloat(f)) + instructions.append(op) + self.add_strings(1) + + elif op == 'DECODE_DOUBLE': + d = self.random.random_float(11) + instructions.push_args(d) + instructions.append(op) + self.add_strings(1) + else: assert False - if op in reads or op in snapshot_reads: + if read_performed and op not in database_reads: self.outstanding_ops.append((self.stack_size, len(instructions)-1)) if args.concurrency == 1 and (op in database_reads or op in database_mutations): diff --git a/bindings/bindingtester/tests/directory.py b/bindings/bindingtester/tests/directory.py index c0037a61a8..6a1ba196cb 100644 --- a/bindings/bindingtester/tests/directory.py +++ b/bindings/bindingtester/tests/directory.py @@ -65,7 +65,7 @@ class DirectoryTest(Test): def setup(self, args): self.dir_index = 0 - self.random = test_util.RandomGenerator(args.max_int_bits) + self.random = test_util.RandomGenerator(args.max_int_bits, args.api_version) def generate(self, args, thread_number): instructions = InstructionSet() diff --git a/bindings/bindingtester/tests/directory_hca.py b/bindings/bindingtester/tests/directory_hca.py index e017c64b57..282a1bbe10 100644 --- a/bindings/bindingtester/tests/directory_hca.py +++ b/bindings/bindingtester/tests/directory_hca.py @@ -38,7 +38,7 @@ class DirectoryHcaTest(Test): self.next_path = 1 def setup(self, args): - self.random = test_util.RandomGenerator(args.max_int_bits) + self.random = test_util.RandomGenerator(args.max_int_bits, args.api_version) self.transactions = ['tr%d' % i for i in range(3)] # SOMEDAY: parameterize this number? self.barrier_num = 0 diff --git a/bindings/bindingtester/tests/test_util.py b/bindings/bindingtester/tests/test_util.py index 1e10dd5a92..fa6948fcfa 100644 --- a/bindings/bindingtester/tests/test_util.py +++ b/bindings/bindingtester/tests/test_util.py @@ -19,15 +19,21 @@ # import random +import uuid import unicodedata +import ctypes +import math import fdb +import fdb.tuple from bindingtester import util +from bindingtester import FDB_API_VERSION class RandomGenerator(object): - def __init__(self, max_int_bits=64): + def __init__(self, max_int_bits=64, api_version=FDB_API_VERSION): self.max_int_bits = max_int_bits + self.api_version = api_version def random_unicode_str(self, length): return u''.join(self.random_unicode_char() for i in range(0, length)) @@ -42,12 +48,23 @@ class RandomGenerator(object): #util.get_logger().debug('generating int (%d): %d - %s' % (num_bits, num, repr(fdb.tuple.pack((num,))))) return num + def random_float(self, exp_bits): + if random.random() < 0.05: + # Choose a special value. + return random.choice([float('-nan'), float('-inf'), -0.0, 0.0, float('inf'), float('nan')]) + else: + # Choose a value from all over the range of acceptable floats for this precision. + sign = -1 if random.random() < 0.5 else 1 + exponent = random.randint(-(1 << (exp_bits-1))-10, (1 << (exp_bits-1) - 1)) + mantissa = random.random() + return sign * math.pow(2, exponent) * mantissa + def random_tuple(self, max_size): size = random.randint(1, max_size) tup = [] for i in range(size): - choice = random.randint(0, 3) + choice = random.randint(0, 8) if choice == 0: tup.append(self.random_int()) elif choice == 1: @@ -56,11 +73,51 @@ class RandomGenerator(object): tup.append(self.random_string(random.randint(0, 100))) elif choice == 3: tup.append(self.random_unicode_str(random.randint(0, 100))) + elif choice == 4: + tup.append(uuid.uuid4()) + elif choice == 5: + b = random.random() < 0.5 + if self.api_version < 500: + tup.append(int(b)) + else: + tup.append(b) + elif choice == 6: + tup.append(fdb.tuple.SingleFloat(self.random_float(8))) + elif choice == 7: + tup.append(self.random_float(11)) + elif choice == 8: + length = random.randint(0, max_size - size) + if length == 0: + tup.append(()) + else: + tup.append(self.random_tuple(length)) else: assert false return tuple(tup) + def random_tuple_list(self, max_size, max_list_size): + size = random.randint(1, max_list_size) + tuples = [] + + for i in range(size): + to_add = self.random_tuple(max_size) + tuples.append(to_add) + if len(to_add) > 1 and random.random() < 0.25: + # Add a smaller one to test prefixes. + smaller_size = random.randint(1, len(to_add)) + tuples.append(to_add[:smaller_size]) + else: + non_empty = filter(lambda (i,x): (isinstance(x, list) or isinstance(x, tuple)) and len(x) > 0, enumerate(to_add)) + if len(non_empty) > 0 and random.random() < 0.25: + # Add a smaller list to test prefixes of nested structures. + idx, choice = random.choice(non_empty) + smaller_size = random.randint(0, len(to_add[idx])) + tuples.append(to_add[:idx] + (choice[:smaller_size],) + to_add[idx+1:]) + + random.shuffle(tuples) + return tuples + def random_range_params(self): if random.random() < 0.75: limit = random.randint(1, 1e3) @@ -87,6 +144,11 @@ class RandomGenerator(object): def random_unicode_char(self): while True: + if random.random() < 0.05: + # Choose one of these special character sequences. + specials = [u'\U0001f4a9', u'\U0001f63c', u'\U0001f3f3\ufe0f\u200d\U0001f308', u'\U0001f1f5\U0001f1f2', u'\uf8ff', + u'\U0002a2b2', u'\u05e9\u05dc\u05d5\u05dd'] + return random.choice(specials) c = random.randint(0, 0xffff) if unicodedata.category(unichr(c))[0] in 'LMNPSZ': return unichr(c) diff --git a/bindings/flow/DirectoryLayer.actor.cpp b/bindings/flow/DirectoryLayer.actor.cpp index 5f3e0b805b..085c3cb51a 100755 --- a/bindings/flow/DirectoryLayer.actor.cpp +++ b/bindings/flow/DirectoryLayer.actor.cpp @@ -26,7 +26,7 @@ namespace FDB { const StringRef DirectoryLayer::HIGH_CONTENTION_KEY = LiteralStringRef("hca"); const StringRef DirectoryLayer::LAYER_KEY = LiteralStringRef("layer"); const StringRef DirectoryLayer::VERSION_KEY = LiteralStringRef("version"); - const uint64_t DirectoryLayer::SUB_DIR_KEY = 0; + const int64_t DirectoryLayer::SUB_DIR_KEY = 0; const uint32_t DirectoryLayer::VERSION[3] = {1, 0, 0}; diff --git a/bindings/flow/DirectoryLayer.h b/bindings/flow/DirectoryLayer.h index e2a8458623..778058075b 100755 --- a/bindings/flow/DirectoryLayer.h +++ b/bindings/flow/DirectoryLayer.h @@ -58,7 +58,7 @@ namespace FDB { static const StringRef HIGH_CONTENTION_KEY; static const StringRef LAYER_KEY; static const StringRef VERSION_KEY; - static const uint64_t SUB_DIR_KEY; + static const int64_t SUB_DIR_KEY; static const uint32_t VERSION[3]; static const StringRef DEFAULT_NODE_SUBSPACE_PREFIX; diff --git a/bindings/flow/HighContentionAllocator.h b/bindings/flow/HighContentionAllocator.h index 602bddaf56..b748e97b74 100755 --- a/bindings/flow/HighContentionAllocator.h +++ b/bindings/flow/HighContentionAllocator.h @@ -28,7 +28,7 @@ namespace FDB { class HighContentionAllocator { public: - HighContentionAllocator(Subspace subspace) : counters(subspace.get(0)), recent(subspace.get(1)) {} + HighContentionAllocator(Subspace subspace) : counters(subspace.get((int64_t)0)), recent(subspace.get((int64_t)1)) {} Future> allocate(Reference const& tr) const; static int64_t windowSize(int64_t start); diff --git a/bindings/flow/Subspace.h b/bindings/flow/Subspace.h index cc3fcb60ae..542a41d8e1 100755 --- a/bindings/flow/Subspace.h +++ b/bindings/flow/Subspace.h @@ -49,6 +49,12 @@ namespace FDB { return pack(t); } + Key packNested(Tuple const& item) const { + Tuple t; + t.appendNested(item); + return pack(t); + } + Key pack(StringRef const& item, bool utf8=false) const { Tuple t; t.append(item, utf8); @@ -65,6 +71,12 @@ namespace FDB { return get(t); } + Subspace getNested(Tuple const& item) const { + Tuple t; + t.appendNested(item); + return get(t); + } + Subspace get(StringRef const& item, bool utf8=false) const { Tuple t; t.append(item, utf8); diff --git a/bindings/flow/Tuple.cpp b/bindings/flow/Tuple.cpp index 25eeb3faf2..f15a7de356 100755 --- a/bindings/flow/Tuple.cpp +++ b/bindings/flow/Tuple.cpp @@ -19,9 +19,50 @@ */ #include "Tuple.h" +#include namespace FDB { + // The floating point operations depend on this using the IEEE 754 standard. + BOOST_STATIC_ASSERT(std::numeric_limits::is_iec559); + BOOST_STATIC_ASSERT(std::numeric_limits::is_iec559); + + const size_t Uuid::SIZE = 16; + + const uint8_t Tuple::NULL_CODE = 0x00; + const uint8_t Tuple::BYTES_CODE = 0x01; + const uint8_t Tuple::STRING_CODE = 0x02; + const uint8_t Tuple::NESTED_CODE = 0x05; + const uint8_t Tuple::INT_ZERO_CODE = 0x14; + const uint8_t Tuple::POS_INT_END = 0x1c; + const uint8_t Tuple::NEG_INT_START = 0x0c; + const uint8_t Tuple::FLOAT_CODE = 0x20; + const uint8_t Tuple::DOUBLE_CODE = 0x21; + const uint8_t Tuple::FALSE_CODE = 0x26; + const uint8_t Tuple::TRUE_CODE = 0x27; + const uint8_t Tuple::UUID_CODE = 0x30; + + static float bigEndianFloat(float orig) { + int32_t big = *(int32_t*)&orig; + big = bigEndian32(big); + return *(float*)&big; + } + + static double bigEndianDouble(double orig) { + int64_t big = *(int64_t*)&orig; + big = bigEndian64(big); + return *(double*)&big; + } + static size_t find_string_terminator(const StringRef data, size_t offset) { + size_t i = offset; + while (i < data.size() - 1 && !(data[i] == (uint8_t)'\x00' && data[i+1] != (uint8_t)'\xff')) { + i += (data[i] == '\x00' ? 2 : 1); + } + + return i; + } + + static size_t find_string_terminator(const Standalone > data, size_t offset ) { size_t i = offset; while (i < data.size() - 1 && !(data[i] == '\x00' && data[i+1] != (uint8_t)'\xff')) { i += (data[i] == '\x00' ? 2 : 1); @@ -30,26 +71,74 @@ namespace FDB { return i; } + // If encoding and the sign bit is 1 (the number is negative), flip all the bits. + // If decoding and the sign bit is 0 (the number is negative), flip all the bits. + // Otherwise, the number is positive, so flip the sign bit. + static void adjust_floating_point(uint8_t *bytes, size_t size, bool encode) { + if((encode && ((uint8_t)(bytes[0] & 0x80) != (uint8_t)0x00)) || (!encode && ((uint8_t)(bytes[0] & 0x80) != (uint8_t)0x80))) { + for(size_t i = 0; i < size; i++) { + bytes[i] ^= (uint8_t)0xff; + } + } else { + bytes[0] ^= (uint8_t)0x80; + } + } + Tuple::Tuple(StringRef const& str) { data.append(data.arena(), str.begin(), str.size()); size_t i = 0; + int depth = 0; while(i < data.size()) { - offsets.push_back(i); + if(depth == 0) offsets.push_back(i); - if(data[i] == '\x01' || data[i] == '\x02') { + if(depth > 0 && data[i] == NULL_CODE) { + if(i + 1 < data.size() && data[i+1] == 0xff) { + // NULL value. + i += 2; + } else { + // Nested terminator. + i += 1; + depth -= 1; + } + } + else if(data[i] == BYTES_CODE || data[i] == STRING_CODE) { i = find_string_terminator(str, i+1) + 1; } - else if(data[i] >= '\x0c' && data[i] <= '\x1c') { - i += abs(data[i] - '\x14') + 1; + else if(data[i] >= NEG_INT_START && data[i] <= POS_INT_END) { + i += abs(data[i] - INT_ZERO_CODE) + 1; } - else if(data[i] == '\x00') { + else if(data[i] == NULL_CODE || data[i] == TRUE_CODE || data[i] == FALSE_CODE) { i += 1; } + else if(data[i] == UUID_CODE) { + i += Uuid::SIZE + 1; + } + else if(data[i] == FLOAT_CODE) { + i += sizeof(float) + 1; + } + else if(data[i] == DOUBLE_CODE) { + i += sizeof(double) + 1; + } + else if(data[i] == NESTED_CODE) { + i += 1; + depth += 1; + } else { throw invalid_tuple_data_type(); } } + + if(depth != 0) { + throw invalid_tuple_data_type(); + } + } + + // Note: this is destructive of the original offsets, so should only + // be used once we are done. + Tuple::Tuple(Standalone> data, std::vector offsets) { + this->data = data; + this->offsets = std::move(offsets); } Tuple Tuple::unpack(StringRef const& str) { @@ -69,7 +158,7 @@ namespace FDB { Tuple& Tuple::append(StringRef const& str, bool utf8) { offsets.push_back(data.size()); - const uint8_t utfChar = uint8_t(utf8 ? '\x02' : '\x01'); + const uint8_t utfChar = utf8 ? STRING_CODE : BYTES_CODE; data.append(data.arena(), &utfChar, 1); size_t lastPos = 0; @@ -88,6 +177,10 @@ namespace FDB { return *this; } + Tuple& Tuple::append( int32_t value ) { + return append((int64_t)value); + } + Tuple& Tuple::append( int64_t value ) { uint64_t swap = value; bool neg = false; @@ -103,19 +196,82 @@ namespace FDB { for ( int i = 0; i < 8; i++ ) { if ( ((uint8_t*)&swap)[i] != (neg ? 255 : 0) ) { - data.push_back( data.arena(), (uint8_t)(20 + (8-i) * (neg ? -1 : 1)) ); + data.push_back( data.arena(), (uint8_t)(INT_ZERO_CODE + (8-i) * (neg ? -1 : 1)) ); data.append( data.arena(), ((const uint8_t *)&swap) + i, 8 - i ); return *this; } } - data.push_back( data.arena(), (uint8_t)'\x14' ); + data.push_back( data.arena(), INT_ZERO_CODE ); + return *this; + } + + Tuple& Tuple::append( bool value ) { + offsets.push_back( data.size() ); + if(value) { + data.push_back( data.arena(), TRUE_CODE ); + } else { + data.push_back( data.arena(), FALSE_CODE ); + } + return *this; + } + + Tuple& Tuple::append( float value ) { + offsets.push_back( data.size() ); + float swap = bigEndianFloat(value); + uint8_t *bytes = (uint8_t*)&swap; + adjust_floating_point(bytes, sizeof(float), true); + + data.push_back( data.arena(), FLOAT_CODE ); + data.append( data.arena(), bytes, sizeof(float) ); + return *this; + } + + Tuple& Tuple::append( double value ) { + offsets.push_back( data.size() ); + double swap = value; + swap = bigEndianDouble(swap); + uint8_t *bytes = (uint8_t*)&swap; + adjust_floating_point(bytes, sizeof(double), true); + + data.push_back( data.arena(), DOUBLE_CODE ); + data.append( data.arena(), bytes, sizeof(double) ); + return *this; + } + + Tuple& Tuple::append( Uuid value ) { + offsets.push_back( data.size() ); + data.push_back( data.arena(), UUID_CODE ); + data.append( data.arena(), value.getData().begin(), Uuid::SIZE ); + return *this; + } + + Tuple& Tuple::appendNested( Tuple const& value ) { + offsets.push_back( data.size() ); + data.push_back( data.arena(), NESTED_CODE ); + + for(size_t i = 0; i < value.size(); i++) { + size_t offset = value.offsets[i]; + size_t next_offset = (i+1 < value.offsets.size() ? value.offsets[i+1] : value.data.size()); + ASSERT(offset < value.data.size()); + ASSERT(next_offset <= value.data.size()); + uint8_t code = value.data[offset]; + if(code == NULL_CODE) { + data.push_back( data.arena(), NULL_CODE ); + data.push_back( data.arena(), 0xff ); + } else { + data.append( data.arena(), value.data.begin() + offset, next_offset - offset); + } + } + + data.push_back( data.arena(), (uint8_t)'\x00'); + return *this; } Tuple& Tuple::appendNull() { offsets.push_back(data.size()); - data.push_back(data.arena(), (uint8_t)'\x00'); + data.push_back(data.arena(), NULL_CODE); return *this; } @@ -126,18 +282,33 @@ namespace FDB { uint8_t code = data[offsets[index]]; - if(code == '\x00') { + if(code == NULL_CODE) { return ElementType::NULL_TYPE; } - else if(code == '\x01') { + else if(code == BYTES_CODE) { return ElementType::BYTES; } - else if(code == '\x02') { + else if(code == STRING_CODE) { return ElementType::UTF8; } - else if(code >= '\x0c' && code <= '\x1c') { + else if(code == NESTED_CODE) { + return ElementType::NESTED; + } + else if(code >= NEG_INT_START && code <= POS_INT_END) { return ElementType::INT; } + else if(code == FLOAT_CODE) { + return ElementType::FLOAT; + } + else if(code == DOUBLE_CODE) { + return ElementType::DOUBLE; + } + else if(code == FALSE_CODE || code == TRUE_CODE) { + return ElementType::BOOL; + } + else if(code == UUID_CODE) { + return ElementType::UUID; + } else { throw invalid_tuple_data_type(); } @@ -149,7 +320,7 @@ namespace FDB { } uint8_t code = data[offsets[index]]; - if(code != '\x01' && code != '\x02') { + if(code != BYTES_CODE && code != STRING_CODE) { throw invalid_tuple_data_type(); } @@ -194,11 +365,11 @@ namespace FDB { ASSERT(offsets[index] < data.size()); uint8_t code = data[offsets[index]]; - if(code < '\x0c' || code > '\x1c') { + if(code < NEG_INT_START || code > POS_INT_END) { throw invalid_tuple_data_type(); } - int8_t len = code - '\x14'; + int8_t len = code - INT_ZERO_CODE; if ( len < 0 ) { len = -len; @@ -217,6 +388,160 @@ namespace FDB { return swap; } + bool Tuple::getBool(size_t index) const { + if(index >= offsets.size()) { + throw invalid_tuple_index(); + } + ASSERT(offsets[index] < data.size()); + uint8_t code = data[offsets[index]]; + if(code == FALSE_CODE) { + return false; + } else if(code == TRUE_CODE) { + return true; + } else { + throw invalid_tuple_data_type(); + } + } + + float Tuple::getFloat(size_t index) const { + if(index >= offsets.size()) { + throw invalid_tuple_index(); + } + ASSERT(offsets[index] < data.size()); + uint8_t code = data[offsets[index]]; + if(code != FLOAT_CODE) { + throw invalid_tuple_data_type(); + } + + float swap; + uint8_t* bytes = (uint8_t*)&swap; + ASSERT(offsets[index] + 1 + sizeof(float) <= data.size()); + swap = *(float*)(data.begin() + offsets[index] + 1); + adjust_floating_point( bytes, sizeof(float), false ); + + return bigEndianFloat(swap); + } + + double Tuple::getDouble(size_t index) const { + if(index >= offsets.size()) { + throw invalid_tuple_index(); + } + ASSERT(offsets[index] < data.size()); + uint8_t code = data[offsets[index]]; + if(code != DOUBLE_CODE) { + throw invalid_tuple_data_type(); + } + + double swap; + uint8_t* bytes = (uint8_t*)&swap; + ASSERT(offsets[index] + 1 + sizeof(double) <= data.size()); + swap = *(double*)(data.begin() + offsets[index] + 1); + adjust_floating_point( bytes, sizeof(double), false ); + + return bigEndianDouble(swap); + } + + Uuid Tuple::getUuid(size_t index) const { + if(index >= offsets.size()) { + throw invalid_tuple_index(); + } + size_t offset = offsets[index]; + ASSERT(offset < data.size()); + uint8_t code = data[offset]; + if(code != UUID_CODE) { + throw invalid_tuple_data_type(); + } + ASSERT(offset + Uuid::SIZE + 1 <= data.size()); + StringRef uuidData(data.begin() + offset + 1, Uuid::SIZE); + return Uuid(uuidData); + } + + Tuple Tuple::getNested(size_t index) const { + if(index >= offsets.size()) { + throw invalid_tuple_index(); + } + size_t offset = offsets[index]; + ASSERT(offset < data.size()); + uint8_t code = data[offset]; + if(code != NESTED_CODE) { + throw invalid_tuple_data_type(); + } + + size_t next_offset = (index + 1 < offsets.size() ? offsets[index+1] : data.size()); + ASSERT(next_offset <= data.size()); + ASSERT(data[next_offset - 1] == (uint8_t)0x00); + Standalone> dest; + dest.reserve(dest.arena(), next_offset - offset); + std::vector dest_offsets; + + size_t i = offset + 1; + int depth = 0; + while(i < next_offset - 1) { + if (depth == 0) dest_offsets.push_back(dest.size()); + uint8_t code = data[i]; + dest.push_back(dest.arena(), code); // Copy over the type code. + if(code == NULL_CODE) { + if(depth > 0) { + if(i + 1 < next_offset - 1 && data[i+1] == 0xff) { + // Null with a tuple nested in the nested tuple. + dest.push_back(dest.arena(), 0xff); + i += 2; + } else { + // Nested terminator. + depth -= 1; + i += 1; + } + } else { + // A null object within the nested tuple. + ASSERT(i + 1 < next_offset - 1); + ASSERT(data[i+1] == 0xff); + i += 2; + } + } + else if(code == BYTES_CODE || code == STRING_CODE) { + size_t next_i = find_string_terminator(data, i+1) + 1; + ASSERT(next_i <= next_offset - 1); + size_t length = next_i - i - 1; + dest.append(dest.arena(), data.begin() + i + 1, length); + i = next_i; + } + else if(code >= NEG_INT_START && code <= POS_INT_END) { + size_t int_size = abs(code - INT_ZERO_CODE); + ASSERT(i + int_size <= next_offset - 1); + dest.append(dest.arena(), data.begin() + i + 1, int_size); + i += int_size + 1; + } + else if(code == TRUE_CODE || code == FALSE_CODE) { + i += 1; + } + else if(code == UUID_CODE) { + ASSERT(i + 1 + Uuid::SIZE <= next_offset - 1); + dest.append(dest.arena(), data.begin() + i + 1, Uuid::SIZE); + i += Uuid::SIZE + 1; + } + else if(code == FLOAT_CODE) { + ASSERT(i + 1 + sizeof(float) <= next_offset - 1); + dest.append(dest.arena(), data.begin() + i + 1, sizeof(float)); + i += sizeof(float) + 1; + } + else if(code == DOUBLE_CODE) { + ASSERT(i + 1 + sizeof(double) <= next_offset - 1); + dest.append(dest.arena(), data.begin() + i + 1, sizeof(double)); + i += sizeof(double) + 1; + } + else if(code == NESTED_CODE) { + i += 1; + depth += 1; + } + else { + throw invalid_tuple_data_type(); + } + } + + // The item may shrink because of escaped nulls that are unespaced. + return Tuple(dest, dest_offsets); + } + KeyRange Tuple::range(Tuple const& tuple) const { VectorRef begin; VectorRef end; @@ -245,4 +570,75 @@ namespace FDB { size_t endPos = end < offsets.size() ? offsets[end] : data.size(); return Tuple(StringRef(data.begin() + offsets[start], endPos - offsets[start])); } + + // Comparisons + int compare(Standalone > const& v1, Standalone > const& v2) { + size_t i = 0; + while(i < v1.size() && i < v2.size()) { + if(v1[i] < v2[i]) { + return -1; + } else if(v1[i] > v2[i]) { + return 1; + } + i += 1; + } + + if(i < v1.size()) { + return 1; + } + if(i < v2.size()) { + return -1; + } + return 0; + } + + bool Tuple::operator==(Tuple const& other) const { + return compare(data, other.data) == 0; + } + bool Tuple::operator!=(Tuple const& other) const { + return compare(data, other.data) != 0; + } + bool Tuple::operator<(Tuple const& other) const { + return compare(data, other.data) < 0; + } + bool Tuple::operator<=(Tuple const& other) const { + return compare(data, other.data) <= 0; + } + bool Tuple::operator>(Tuple const& other) const { + return compare(data, other.data) > 0; + } + bool Tuple::operator>=(Tuple const& other) const { + return compare(data, other.data) >= 0; + } + + // UUID implementation + Uuid::Uuid(const StringRef& data) { + if (data.size() != Uuid::SIZE) { + throw invalid_uuid_size(); + } + this->data = data; + } + + StringRef Uuid::getData() const { + return data; + } + + bool Uuid::operator==(Uuid const& other) const { + return data == other.data; + } + bool Uuid::operator!=(Uuid const& other) const { + return data != other.data; + } + bool Uuid::operator<(Uuid const& other) const { + return data < other.data; + } + bool Uuid::operator<=(Uuid const& other) const { + return data <= other.data; + } + bool Uuid::operator>(Uuid const& other) const { + return data > other.data; + } + bool Uuid::operator>=(Uuid const& other) const { + return data >= other.data; + } } diff --git a/bindings/flow/Tuple.h b/bindings/flow/Tuple.h index 890627f35e..e9ccda9ca2 100755 --- a/bindings/flow/Tuple.h +++ b/bindings/flow/Tuple.h @@ -26,6 +26,24 @@ #include "bindings/flow/fdb_flow.h" namespace FDB { + struct Uuid { + const static size_t SIZE; + + Uuid(StringRef const& data); + + StringRef getData() const; + + // Comparisons + bool operator==(Uuid const& other) const; + bool operator!=(Uuid const& other) const; + bool operator<(Uuid const& other) const; + bool operator<=(Uuid const& other) const; + bool operator>(Uuid const& other) const; + bool operator>=(Uuid const& other) const; + private: + Standalone data; + }; + struct Tuple { Tuple() {} @@ -33,7 +51,13 @@ namespace FDB { Tuple& append(Tuple const& tuple); Tuple& append(StringRef const& str, bool utf8=false); + Tuple& append(int32_t); Tuple& append(int64_t); + Tuple& append(bool); + Tuple& append(float); + Tuple& append(double); + Tuple& append(Uuid); + Tuple& appendNested(Tuple const&); Tuple& appendNull(); StringRef pack() const { return StringRef(data.begin(), data.size()); } @@ -43,7 +67,7 @@ namespace FDB { return append(t); } - enum ElementType { NULL_TYPE, INT, BYTES, UTF8 }; + enum ElementType { NULL_TYPE, INT, BYTES, UTF8, BOOL, FLOAT, DOUBLE, UUID, NESTED }; // this is number of elements, not length of data size_t size() const { return offsets.size(); } @@ -51,13 +75,40 @@ namespace FDB { ElementType getType(size_t index) const; Standalone getString(size_t index) const; int64_t getInt(size_t index) const; + bool getBool(size_t index) const; + float getFloat(size_t index) const; + double getDouble(size_t index) const; + Uuid getUuid(size_t index) const; + Tuple getNested(size_t index) const; KeyRange range(Tuple const& tuple = Tuple()) const; Tuple subTuple(size_t beginIndex, size_t endIndex = std::numeric_limits::max()) const; + // Comparisons + bool operator==(Tuple const& other) const; + bool operator!=(Tuple const& other) const; + bool operator<(Tuple const& other) const; + bool operator<=(Tuple const& other) const; + bool operator>(Tuple const& other) const; + bool operator>=(Tuple const& other) const; + private: + static const uint8_t NULL_CODE; + static const uint8_t BYTES_CODE; + static const uint8_t STRING_CODE; + static const uint8_t NESTED_CODE; + static const uint8_t INT_ZERO_CODE; + static const uint8_t POS_INT_END; + static const uint8_t NEG_INT_START; + static const uint8_t FLOAT_CODE; + static const uint8_t DOUBLE_CODE; + static const uint8_t FALSE_CODE; + static const uint8_t TRUE_CODE; + static const uint8_t UUID_CODE; + Tuple(const StringRef& data); + Tuple(Standalone> data, std::vector offsets); Standalone> data; std::vector offsets; }; diff --git a/bindings/flow/tester/DirectoryTester.actor.cpp b/bindings/flow/tester/DirectoryTester.actor.cpp index 713359a2ef..b8fe7b536a 100644 --- a/bindings/flow/tester/DirectoryTester.actor.cpp +++ b/bindings/flow/tester/DirectoryTester.actor.cpp @@ -405,7 +405,7 @@ struct DirectoryExistsFunc : InstructionFunc { logOp(format("exists %s: %d", pathToString(combinePaths(directory->getPath(), path)).c_str(), result)); } - data->stack.push(Tuple().append(result ? 1 : 0).pack()); + data->stack.push(Tuple().append((int64_t)(result ? 1 : 0)).pack()); return Void(); } }; @@ -468,7 +468,7 @@ struct DirectoryContainsFunc : InstructionFunc { ACTOR static Future call(Reference data, Reference instruction) { Tuple key = wait(data->stack.waitAndPop()); bool result = data->directoryData.subspace()->contains(key.getString(0)); - data->stack.push(Tuple().append(result ? 1 : 0).pack()); + data->stack.push(Tuple().append((int64_t)(result ? 1 : 0)).pack()); return Void(); } @@ -500,7 +500,7 @@ struct DirectoryLogSubspaceFunc : InstructionFunc { ACTOR static Future call(Reference data, Reference instruction) { Tuple prefix = wait(data->stack.waitAndPop()); Tuple tuple; - tuple.append(data->directoryData.directoryListIndex); + tuple.append((int64_t)data->directoryData.directoryListIndex); instruction->tr->set(Subspace(tuple, prefix.getString(0)).key(), data->directoryData.subspace()->key()); return Void(); @@ -526,7 +526,7 @@ struct DirectoryLogDirectoryFunc : InstructionFunc { } } - Subspace logSubspace(Tuple().append(data->directoryData.directoryListIndex), prefix.getString(0)); + Subspace logSubspace(Tuple().append((int64_t)(data->directoryData.directoryListIndex)), prefix.getString(0)); Tuple pathTuple; for(auto &p : directory->getPath()) { @@ -535,7 +535,7 @@ struct DirectoryLogDirectoryFunc : InstructionFunc { instruction->tr->set(logSubspace.pack(LiteralStringRef("path"), true), pathTuple.pack()); instruction->tr->set(logSubspace.pack(LiteralStringRef("layer"), true), Tuple().append(directory->getLayer()).pack()); - instruction->tr->set(logSubspace.pack(LiteralStringRef("exists"), true), Tuple().append(exists ? 1 : 0).pack()); + instruction->tr->set(logSubspace.pack(LiteralStringRef("exists"), true), Tuple().append((int64_t)(exists ? 1 : 0)).pack()); instruction->tr->set(logSubspace.pack(LiteralStringRef("children"), true), childrenTuple.pack()); return Void(); diff --git a/bindings/flow/tester/Tester.actor.cpp b/bindings/flow/tester/Tester.actor.cpp index 499a116413..192989d36d 100644 --- a/bindings/flow/tester/Tester.actor.cpp +++ b/bindings/flow/tester/Tester.actor.cpp @@ -24,7 +24,6 @@ #include "bindings/flow/FDBLoanerTypes.h" #include "Tester.actor.h" - #ifdef __linux__ #include #endif @@ -102,6 +101,22 @@ std::string tupleToString(Tuple const& tuple) { else if(type == Tuple::INT) { str += format("%ld", tuple.getInt(i)); } + else if(type == Tuple::FLOAT) { + str += format("%f", tuple.getFloat(i)); + } + else if(type == Tuple::DOUBLE) { + str += format("%f", tuple.getDouble(i)); + } + else if(type == Tuple::BOOL) { + str += tuple.getBool(i) ? "true" : "false"; + } + else if(type == Tuple::UUID) { + Uuid u = tuple.getUuid(i); + str += format("%016llx%016llx", *(uint64_t*)u.getData().begin(), *(uint64_t*)(u.getData().begin() + 8)); + } + else if(type == Tuple::NESTED) { + str += tupleToString(tuple.getNested(i)); + } else { ASSERT(false); } @@ -377,8 +392,8 @@ struct LogStackFunc : InstructionFunc { try { for(auto it : entries) { Tuple tk; - tk.append(it.first); - tk.append(it.second.index); + tk.append((int64_t)it.first); + tk.append((int64_t)it.second.index); state Standalone pk = tk.pack().withPrefix(prefix); Standalone pv = wait(it.second.value); tr->set(pk, pv.substr(0, std::min(pv.size(), 40000))); @@ -1027,6 +1042,21 @@ struct TuplePackFunc : InstructionFunc { else if(type == Tuple::UTF8) { tuple.append(itemTuple.getString(0), true); } + else if(type == Tuple::FLOAT) { + tuple << itemTuple.getFloat(0); + } + else if(type == Tuple::DOUBLE) { + tuple << itemTuple.getDouble(0); + } + else if(type == Tuple::BOOL) { + tuple << itemTuple.getBool(0); + } + else if(type == Tuple::UUID) { + tuple << itemTuple.getUuid(0); + } + else if(type == Tuple::NESTED) { + tuple.appendNested(itemTuple.getNested(0)); + } else { ASSERT(false); } @@ -1083,7 +1113,7 @@ struct TupleRangeFunc : InstructionFunc { return Void(); state Tuple tuple; - state int i = 0; + state size_t i = 0; for (; i < items1.size(); ++i) { Standalone str = wait(items1[i].value); Tuple itemTuple = Tuple::unpack(str); @@ -1101,6 +1131,21 @@ struct TupleRangeFunc : InstructionFunc { else if(type == Tuple::UTF8) { tuple.append(itemTuple.getString(0), true); } + else if(type == Tuple::FLOAT) { + tuple << itemTuple.getFloat(0); + } + else if(type == Tuple::DOUBLE) { + tuple << itemTuple.getDouble(0); + } + else if(type == Tuple::BOOL) { + tuple << itemTuple.getBool(0); + } + else if(type == Tuple::UUID) { + tuple << itemTuple.getUuid(0); + } + else if(type == Tuple::NESTED) { + tuple.appendNested(itemTuple.getNested(0)); + } else { ASSERT(false); } @@ -1120,6 +1165,141 @@ struct TupleRangeFunc : InstructionFunc { const char* TupleRangeFunc::name = "TUPLE_RANGE"; REGISTER_INSTRUCTION_FUNC(TupleRangeFunc); +// TUPLE_SORT +struct TupleSortFunc : InstructionFunc { + static const char* name; + + ACTOR static Future call(Reference data, Reference instruction) { + state std::vector items = data->stack.pop(); + if (items.size() != 1) + return Void(); + + Standalone s1 = wait(items[0].value); + state int64_t count = Tuple::unpack(s1).getInt(0); + + state std::vector items1 = data->stack.pop(count); + if (items1.size() != count) + return Void(); + + state std::vector tuples; + state size_t i = 0; + for(; i < items1.size(); i++) { + Standalone value = wait(items1[i].value); + tuples.push_back(Tuple::unpack(value)); + } + + std::sort(tuples.begin(), tuples.end()); + for(Tuple const& t : tuples) { + data->stack.push(t.pack()); + } + + return Void(); + } +}; +const char* TupleSortFunc::name = "TUPLE_SORT"; +REGISTER_INSTRUCTION_FUNC(TupleSortFunc); + +// ENCODE_FLOAT +struct EncodeFloatFunc : InstructionFunc { + static const char* name; + + ACTOR static Future call(Reference data, Reference instruction) { + std::vector items = data->stack.pop(); + if (items.size() != 1) + return Void(); + + Standalone s1 = wait(items[0].value); + Standalone fBytes = Tuple::unpack(s1).getString(0); + ASSERT(fBytes.size() == 4); + + int32_t intVal = *(int32_t*)fBytes.begin(); + intVal = bigEndian32(intVal); + float fVal = *(float*)&intVal; + + Tuple t; + t.append(fVal); + data->stack.push(t.pack()); + + return Void(); + } +}; +const char* EncodeFloatFunc::name = "ENCODE_FLOAT"; +REGISTER_INSTRUCTION_FUNC(EncodeFloatFunc); + +// ENCODE_DOUBLE +struct EncodeDoubleFunc : InstructionFunc { + static const char* name; + + ACTOR static Future call(Reference data, Reference instruction) { + std::vector items = data->stack.pop(); + if (items.size() != 1) + return Void(); + + Standalone s1 = wait(items[0].value); + Standalone dBytes = Tuple::unpack(s1).getString(0); + ASSERT(dBytes.size() == 8); + + int64_t intVal = *(int64_t*)dBytes.begin(); + intVal = bigEndian64(intVal); + double dVal = *(double*)&intVal; + + Tuple t; + t.append(dVal); + data->stack.push(t.pack()); + + return Void(); + } +}; +const char* EncodeDoubleFunc::name = "ENCODE_DOUBLE"; +REGISTER_INSTRUCTION_FUNC(EncodeDoubleFunc); + +// DECODE_FLOAT +struct DecodeFloatFunc : InstructionFunc { + static const char* name; + + ACTOR static Future call(Reference data, Reference instruction) { + std::vector items = data->stack.pop(); + if (items.size() != 1) + return Void(); + + Standalone s1 = wait(items[0].value); + float fVal = Tuple::unpack(s1).getFloat(0); + int32_t intVal = *(int32_t*)&fVal; + intVal = bigEndian32(intVal); + + Tuple t; + t.append(StringRef((uint8_t*)&intVal, 4), false); + data->stack.push(t.pack()); + + return Void(); + } +}; +const char* DecodeFloatFunc::name = "DECODE_FLOAT"; +REGISTER_INSTRUCTION_FUNC(DecodeFloatFunc); + +// DECODE_DOUBLE +struct DecodeDoubleFunc : InstructionFunc { + static const char* name; + + ACTOR static Future call(Reference data, Reference instruction) { + std::vector items = data->stack.pop(); + if (items.size() != 1) + return Void(); + + Standalone s1 = wait(items[0].value); + double dVal = Tuple::unpack(s1).getDouble(0); + int64_t intVal = *(int64_t*)&dVal; + intVal = bigEndian64(intVal); + + Tuple t; + t.append(StringRef((uint8_t*)&intVal, 8), false); + data->stack.push(t.pack()); + return Void(); + } +}; +const char* DecodeDoubleFunc::name = "DECODE_DOUBLE"; +REGISTER_INSTRUCTION_FUNC(DecodeDoubleFunc); + // Thread Operations // START_THREAD struct StartThreadFunc : InstructionFunc { diff --git a/bindings/go/src/_stacktester/stacktester.go b/bindings/go/src/_stacktester/stacktester.go index dab5a0c9ca..81a1c9e18b 100644 --- a/bindings/go/src/_stacktester/stacktester.go +++ b/bindings/go/src/_stacktester/stacktester.go @@ -22,11 +22,14 @@ package main import ( "bytes" + "encoding/binary" + "encoding/hex" "fdb" "fdb/tuple" "log" "fmt" "os" + "sort" "strings" "sync" "runtime" @@ -40,6 +43,21 @@ const verbose bool = false var trMap = map[string]fdb.Transaction {} var trMapLock = sync.RWMutex{} +// Make tuples sortable by byte-order +type byBytes []tuple.Tuple + +func (b byBytes) Len() int { + return len(b) +} + +func (b byBytes) Swap(i, j int) { + b[i], b[j] = b[j], b[i] +} + +func (b byBytes) Less(i, j int) bool { + return bytes.Compare(b[i].Pack(), b[j].Pack()) < 0 +} + func int64ToBool(i int64) bool { switch i { case 0: @@ -83,7 +101,10 @@ func (sm *StackMachine) waitAndPop() (ret stackEntry) { ret, sm.stack = sm.stack[len(sm.stack) - 1], sm.stack[:len(sm.stack) - 1] switch el := ret.item.(type) { - case int64, []byte, string: + case []byte: + ret.item = el + case int64, string, bool, tuple.UUID, float32, float64, tuple.Tuple: + ret.item = el case fdb.Key: ret.item = []byte(el) case fdb.FutureNil: @@ -145,6 +166,40 @@ func (sm *StackMachine) store(idx int, item interface{}) { sm.stack = append(sm.stack, stackEntry{item, idx}) } +func tupleToString(t tuple.Tuple) string { + var buffer bytes.Buffer + buffer.WriteByte('(') + for i, el := range t { + if i > 0 { + buffer.WriteString(", ") + } + switch el := el.(type) { + case int64: + buffer.WriteString(fmt.Sprintf("%d", el)) + case []byte: + buffer.WriteString(fmt.Sprintf("%+q", string(el))) + case string: + buffer.WriteString(fmt.Sprintf("%+q", el)) + case bool: + buffer.WriteString(fmt.Sprintf("%t", el)) + case tuple.UUID: + buffer.WriteString(hex.EncodeToString(el[:])) + case float32: + buffer.WriteString(fmt.Sprintf("%f", el)) + case float64: + buffer.WriteString(fmt.Sprintf("%f", el)) + case nil: + buffer.WriteString("nil") + case tuple.Tuple: + buffer.WriteString(tupleToString(el)) + default: + log.Fatalf("Don't know how to stringify tuple elemement %v %T\n", el, el) + } + } + buffer.WriteByte(')') + return buffer.String() +} + func (sm *StackMachine) dumpStack() { for i := len(sm.stack) - 1; i >= 0; i-- { fmt.Printf(" %d.", sm.stack[i].idx) @@ -164,6 +219,16 @@ func (sm *StackMachine) dumpStack() { fmt.Printf(" %+q", string(el)) case string: fmt.Printf(" %+q", el) + case bool: + fmt.Printf(" %t", el) + case tuple.Tuple: + fmt.Printf(" %s", tupleToString(el)) + case tuple.UUID: + fmt.Printf(" %s", hex.EncodeToString(el[:])) + case float32: + fmt.Printf(" %f", el) + case float64: + fmt.Printf(" %f", el) case nil: fmt.Printf(" nil") default: @@ -561,6 +626,39 @@ func (sm *StackMachine) processInst(idx int, inst tuple.Tuple) { for _, el := range(t) { sm.store(idx, []byte(tuple.Tuple{el}.Pack())) } + case op == "TUPLE_SORT": + count := sm.waitAndPop().item.(int64) + tuples := make([]tuple.Tuple, count) + for i := 0; i < int(count); i++ { + tuples[i], e = tuple.Unpack(fdb.Key(sm.waitAndPop().item.([]byte))) + if e != nil { + panic(e) + } + } + sort.Sort(byBytes(tuples)) + for _, t := range tuples { + sm.store(idx, t.Pack()) + } + case op == "ENCODE_FLOAT": + val_bytes := sm.waitAndPop().item.([]byte) + var val float32 + binary.Read(bytes.NewBuffer(val_bytes), binary.BigEndian, &val) + sm.store(idx, val) + case op == "ENCODE_DOUBLE": + val_bytes := sm.waitAndPop().item.([]byte) + var val float64 + binary.Read(bytes.NewBuffer(val_bytes), binary.BigEndian, &val) + sm.store(idx, val) + case op == "DECODE_FLOAT": + val := sm.waitAndPop().item.(float32) + var ibuf bytes.Buffer + binary.Write(&ibuf, binary.BigEndian, val) + sm.store(idx, ibuf.Bytes()) + case op == "DECODE_DOUBLE": + val := sm.waitAndPop().item.(float64) + var ibuf bytes.Buffer + binary.Write(&ibuf, binary.BigEndian, val) + sm.store(idx, ibuf.Bytes()) case op == "TUPLE_RANGE": var t tuple.Tuple count := sm.waitAndPop().item.(int64) @@ -618,7 +716,8 @@ func (sm *StackMachine) processInst(idx int, inst tuple.Tuple) { case op == "ATOMIC_OP": opname := strings.Replace(strings.Title(strings.Replace(strings.ToLower(sm.waitAndPop().item.(string)), "_", " ", -1)), " ", "", -1) key := fdb.Key(sm.waitAndPop().item.([]byte)) - value := sm.waitAndPop().item.([]byte) + ival := sm.waitAndPop().item + value := ival.([]byte) sm.executeMutation(t, func (tr fdb.Transaction) (interface{}, error) { reflect.ValueOf(tr).MethodByName(opname).Call([]reflect.Value{reflect.ValueOf(key), reflect.ValueOf(value)}) return nil, nil diff --git a/bindings/go/src/fdb/tuple/tuple.go b/bindings/go/src/fdb/tuple/tuple.go index 2cec4c2a64..1f1c5959da 100644 --- a/bindings/go/src/fdb/tuple/tuple.go +++ b/bindings/go/src/fdb/tuple/tuple.go @@ -47,7 +47,7 @@ import ( // result in a runtime panic). // // The valid types for TupleElement are []byte (or fdb.KeyConvertible), string, -// int64 (or int), and nil. +// int64 (or int), float, double, bool, UUID, Tuple, and nil. type TupleElement interface{} // Tuple is a slice of objects that can be encoded as FoundationDB tuples. If @@ -59,6 +59,28 @@ type TupleElement interface{} // packing T (modulo type normalization to []byte and int64). type Tuple []TupleElement +// UUID wraps a basic byte array as a UUID. We do not provide any special +// methods for accessing or generating the UUID, but as Go does not provide +// a built-in UUID type, this simple wrapper allows for other libraries +// to write the output of their UUID type as a 16-byte array into +// an instance of this type. +type UUID [16]byte + +// Type codes: These prefix the different elements in a packed Tuple +// to indicate what type they are. +const nilCode = 0x00 +const bytesCode = 0x01 +const stringCode = 0x02 +const nestedCode = 0x05 +const intZeroCode = 0x14 +const posIntEnd = 0x1c +const negIntStart = 0x0c +const floatCode = 0x20 +const doubleCode = 0x21 +const falseCode = 0x26 +const trueCode = 0x27 +const uuidCode = 0x30 + var sizeLimits = []uint64{ 1 << (0 * 8) - 1, 1 << (1 * 8) - 1, @@ -71,6 +93,18 @@ var sizeLimits = []uint64{ 1 << (8 * 8) - 1, } +func adjustFloatBytes(b []byte, encode bool) { + if (encode && b[0] & 0x80 != 0x00) || (!encode && b[0] & 0x80 == 0x00) { + // Negative numbers: flip all of the bytes. + for i := 0; i < len(b); i++ { + b[i] = b[i] ^ 0xff + } + } else { + // Positive number: flip just the sign bit. + b[0] = b[0] ^ 0x80 + } +} + func encodeBytes(buf *bytes.Buffer, code byte, b []byte) { buf.WriteByte(code) buf.Write(bytes.Replace(b, []byte{0x00}, []byte{0x00, 0xFF}, -1)) @@ -97,7 +131,7 @@ func encodeInt(buf *bytes.Buffer, i int64) { switch { case i > 0: n = bisectLeft(uint64(i)) - buf.WriteByte(byte(0x14+n)) + buf.WriteByte(byte(intZeroCode+n)) binary.Write(&ibuf, binary.BigEndian, i) case i < 0: n = bisectLeft(uint64(-i)) @@ -108,35 +142,86 @@ func encodeInt(buf *bytes.Buffer, i int64) { buf.Write(ibuf.Bytes()[8-n:]) } +func encodeFloat(buf *bytes.Buffer, f float32) { + var ibuf bytes.Buffer + binary.Write(&ibuf, binary.BigEndian, f) + buf.WriteByte(floatCode) + out := ibuf.Bytes() + adjustFloatBytes(out, true) + buf.Write(out) +} + +func encodeDouble(buf *bytes.Buffer, d float64) { + var ibuf bytes.Buffer + binary.Write(&ibuf, binary.BigEndian, d) + buf.WriteByte(doubleCode) + out := ibuf.Bytes() + adjustFloatBytes(out, true) + buf.Write(out) +} + +func encodeUUID(buf *bytes.Buffer, u UUID) { + buf.WriteByte(uuidCode) + buf.Write(u[:]) +} + +func encodeTuple(buf *bytes.Buffer, t Tuple, nested bool) { + if nested { + buf.WriteByte(nestedCode) + } + + for i, e := range(t) { + switch e := e.(type) { + case Tuple: + encodeTuple(buf, e, true) + case nil: + buf.WriteByte(nilCode) + if nested { + buf.WriteByte(0xff) + } + case int64: + encodeInt(buf, e) + case int: + encodeInt(buf, int64(e)) + case []byte: + encodeBytes(buf, bytesCode, e) + case fdb.KeyConvertible: + encodeBytes(buf, bytesCode, []byte(e.FDBKey())) + case string: + encodeBytes(buf, stringCode, []byte(e)) + case float32: + encodeFloat(buf, e) + case float64: + encodeDouble(buf, e) + case bool: + if e { + buf.WriteByte(trueCode) + } else { + buf.WriteByte(falseCode) + } + case UUID: + encodeUUID(buf, e) + default: + panic(fmt.Sprintf("unencodable element at index %d (%v, type %T)", i, t[i], t[i])) + } + } + + if nested { + buf.WriteByte(0x00) + } +} + // Pack returns a new byte slice encoding the provided tuple. Pack will panic if // the tuple contains an element of any type other than []byte, -// fdb.KeyConvertible, string, int64, int or nil. +// fdb.KeyConvertible, string, int64, int, float32, float64, bool, tuple.UUID, +// nil, or a Tuple with elements of valid types. // // Tuple satisfies the fdb.KeyConvertible interface, so it is not necessary to // call Pack when using a Tuple with a FoundationDB API function that requires a // key. func (t Tuple) Pack() []byte { buf := new(bytes.Buffer) - - for i, e := range(t) { - switch e := e.(type) { - case nil: - buf.WriteByte(0x00) - case int64: - encodeInt(buf, e) - case int: - encodeInt(buf, int64(e)) - case []byte: - encodeBytes(buf, 0x01, e) - case fdb.KeyConvertible: - encodeBytes(buf, 0x01, []byte(e.FDBKey())) - case string: - encodeBytes(buf, 0x02, []byte(e)) - default: - panic(fmt.Sprintf("unencodable element at index %d (%v, type %T)", i, t[i], t[i])) - } - } - + encodeTuple(buf, t, false) return buf.Bytes() } @@ -168,13 +253,13 @@ func decodeString(b []byte) (string, int) { } func decodeInt(b []byte) (int64, int) { - if b[0] == 0x14 { + if b[0] == intZeroCode { return 0, 1 } var neg bool - n := int(b[0]) - 20 + n := int(b[0]) - intZeroCode if n < 0 { n = -n neg = true @@ -194,9 +279,31 @@ func decodeInt(b []byte) (int64, int) { return ret, n+1 } -// Unpack returns the tuple encoded by the provided byte slice, or an error if -// the key does not correctly encode a FoundationDB tuple. -func Unpack(b []byte) (Tuple, error) { +func decodeFloat(b []byte) (float32, int) { + bp := make([]byte, 4) + copy(bp, b[1:]) + adjustFloatBytes(bp, false) + var ret float32 + binary.Read(bytes.NewBuffer(bp), binary.BigEndian, &ret) + return ret, 5 +} + +func decodeDouble(b []byte) (float64, int) { + bp := make([]byte, 8) + copy(bp, b[1:]) + adjustFloatBytes(bp, false) + var ret float64 + binary.Read(bytes.NewBuffer(bp), binary.BigEndian, &ret) + return ret, 9 +} + +func decodeUUID(b []byte) (UUID, int) { + var u UUID + copy(u[:], b[1:]) + return u, 17 +} + +func decodeTuple(b []byte, nested bool) (Tuple, int, error) { var t Tuple var i int @@ -206,24 +313,66 @@ func Unpack(b []byte) (Tuple, error) { var off int switch { - case b[i] == 0x00: - el = nil - off = 1 - case b[i] == 0x01: + case b[i] == nilCode: + if !nested { + el = nil + off = 1 + } else if i + 1 < len(b) && b[i+1] == 0xff { + el = nil + off = 2 + } else { + return t, i+1, nil + } + case b[i] == bytesCode: el, off = decodeBytes(b[i:]) - case b[i] == 0x02: + case b[i] == stringCode: el, off = decodeString(b[i:]) - case 0x0c <= b[i] && b[i] <= 0x1c: + case negIntStart <= b[i] && b[i] <= posIntEnd: el, off = decodeInt(b[i:]) + case b[i] == floatCode: + if i + 5 > len(b) { + return nil, i, fmt.Errorf("insufficient bytes to decode float starting at position %d of byte array for tuple", i) + } + el, off = decodeFloat(b[i:]) + case b[i] == doubleCode: + if i + 9 > len(b) { + return nil, i, fmt.Errorf("insufficient bytes to decode double starting at position %d of byte array for tuple", i) + } + el, off = decodeDouble(b[i:]) + case b[i] == trueCode: + el = true + off = 1 + case b[i] == falseCode: + el = false + off = 1 + case b[i] == uuidCode: + if i + 17 > len(b) { + return nil, i, fmt.Errorf("insufficient bytes to decode UUID starting at position %d of byte array for tuple", i) + } + el, off = decodeUUID(b[i:]) + case b[i] == nestedCode: + var err error + el, off, err = decodeTuple(b[i+1:], true) + if err != nil { + return nil, i, err + } + off += 1 default: - return nil, fmt.Errorf("unable to decode tuple element with unknown typecode %02x", b[i]) + return nil, i, fmt.Errorf("unable to decode tuple element with unknown typecode %02x", b[i]) } t = append(t, el) i += off } - return t, nil + return t, i, nil +} + +// Unpack returns the tuple encoded by the provided byte slice, or an error if +// the key does not correctly encode a FoundationDB tuple. +func Unpack(b []byte) (Tuple, error) { + t, _, err := decodeTuple(b, false) + return t, err } // FDBKey returns the packed representation of a Tuple, and allows Tuple to diff --git a/bindings/java/src-completable/main/com/apple/cie/foundationdb/FDBDatabase.java b/bindings/java/src-completable/main/com/apple/cie/foundationdb/FDBDatabase.java index 25bbd1b426..be4b8fac98 100644 --- a/bindings/java/src-completable/main/com/apple/cie/foundationdb/FDBDatabase.java +++ b/bindings/java/src-completable/main/com/apple/cie/foundationdb/FDBDatabase.java @@ -32,7 +32,7 @@ import com.apple.cie.foundationdb.async.AsyncUtil; class FDBDatabase extends DefaultDisposableImpl implements Database, Disposable, OptionConsumer { private DatabaseOptions options; - private Executor executor; + private final Executor executor; protected FDBDatabase(long cPtr, Executor executor) { super(cPtr); diff --git a/bindings/java/src-completable/main/com/apple/cie/foundationdb/tuple/IterableComparator.java b/bindings/java/src-completable/main/com/apple/cie/foundationdb/tuple/IterableComparator.java new file mode 100644 index 0000000000..d09bf976de --- /dev/null +++ b/bindings/java/src-completable/main/com/apple/cie/foundationdb/tuple/IterableComparator.java @@ -0,0 +1,107 @@ +/* + * IterableComparator.java + * + * This source file is part of the FoundationDB open source project + * + * Copyright 2013-2018 Apple Inc. and the FoundationDB project authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.apple.cie.foundationdb.tuple; + +import java.util.Comparator; +import java.util.Iterator; + +/** + * A {@link Tuple}-compatible {@link Comparator} that will sort {@link Iterable}s + * in a manner that is consistent with the byte-representation of {@link Tuple}s. + * In particular, if one has two {@link Tuple}s, {@code tuple1} and {@code tuple2}, + * it is the case that: + * + * 
+ * {@code
+ *    tuple1.compareTo(tuple2)
+ *      == new IterableComparator().compare(tuple1, tuple2)
+ *      == new IterableComparator().compare(tuple1.getItems(), tuple2.getItems()),
+ *      == ByteArrayUtil.compareUnsigned(tuple1.pack(), tuple2.pack())}
+ *
+ * + *

+ * The individual elements of the {@link Iterable} must be of a type that can + * be serialized by a {@link Tuple}. For items of identical types, they will be + * sorted in a way that is consistent with their natural ordering with a few + * caveats: + *

+ * + *
    + *
  • + * For floating point types, negative NaN values are sorted before all regular values, and + * positive NaN values are sorted after all regular values. + *
    • + *
  • + * Single-precision floating point numbers are sorted before + * all double-precision floating point numbers. + *
    • + *
  • + * UUIDs are sorted by their unsigned Big-Endian byte representation + * rather than their signed byte representation (which is the behavior of + * {@link java.util.UUID#compareTo(java.util.UUID) UUID.compareTo()}). + *
    • + *
  • Strings are sorted explicitly by their UTF-8 byte representation
    • + *
  • Nested {@link Tuple}s and {@link java.util.List List}s are sorted element-wise.
    • + *
+ */ +public class IterableComparator implements Comparator> { + /** + * Creates a new {@code IterableComparator}. This {@link Comparator} has + * no internal state. + */ + public IterableComparator() {} + + /** + * Compare two {@link Iterable}s in a way consistent with their + * byte representation. This is done element-wise and is consistent + * with a number of other ways of sorting {@link Tuple}s. This will + * raise an {@link IllegalArgumentException} if any of the items + * of either {@link Iterable} cannot be serialized by a {@link Tuple}. + * + * @param iterable1 the first {@link Iterable} of items + * @param iterable2 the second {@link Iterable} of items + * @return a negative number if the first iterable would sort before the second + * when serialized, a positive number if the opposite is true, and zero + * if the two are equal + */ + @Override + public int compare(Iterable iterable1, Iterable iterable2) { + Iterator i1 = iterable1.iterator(); + Iterator i2 = iterable2.iterator(); + + while(i1.hasNext() && i2.hasNext()) { + int itemComp = TupleUtil.compareItems(i1.next(), i2.next()); + if(itemComp != 0) { + return itemComp; + } + } + + if(i1.hasNext()) { + // iterable2 is a prefix of iterable1. + return 1; + } + if(i2.hasNext()) { + // iterable1 is a prefix of iterable2. + return -1; + } + return 0; + } +} diff --git a/bindings/java/src-completable/main/com/apple/cie/foundationdb/tuple/Tuple.java b/bindings/java/src-completable/main/com/apple/cie/foundationdb/tuple/Tuple.java index e70586a83a..90782637e7 100644 --- a/bindings/java/src-completable/main/com/apple/cie/foundationdb/tuple/Tuple.java +++ b/bindings/java/src-completable/main/com/apple/cie/foundationdb/tuple/Tuple.java @@ -20,12 +20,15 @@ package com.apple.cie.foundationdb.tuple; +import java.math.BigInteger; import java.util.ArrayList; import java.util.Arrays; +import java.util.Collection; import java.util.Collections; import java.util.Iterator; import java.util.LinkedList; import java.util.List; +import java.util.UUID; import java.util.stream.Collectors; import java.util.stream.Stream; @@ -39,26 +42,34 @@ import com.apple.cie.foundationdb.Range; * ideal for building a variety of higher-level data models.
*

Types

* A {@code Tuple} can - * contain byte arrays ({@code byte[]}), {@link String}s, {@link Number}s, and {@code null}. All - * {@code Number}s will be converted to a {@code long} integral value, so all - * floating point information will be lost and their range will be constrained to the range - * [{@code 2^63-1}, {@code -2^63}]. Note that for numbers outside this range the way that Java + * contain byte arrays ({@code byte[]}), {@link String}s, {@link Number}s, {@link UUID}s, + * {@code boolean}s, {@link List}s, other {@code Tuple}s, and {@code null}. + * {@link Float} and {@link Double} instances will be serialized as single- and double-precision + * numbers respectively, and {@link BigInteger}s within the range [{@code -2^2040+1}, + * {@code 2^2040-1}] are serialized without loss of precision (those outside the range + * will raise an {@link IllegalArgumentException}). All other {@code Number}s will be converted to + * a {@code long} integral value, so the range will be constrained to + * [{@code -2^63}, {@code 2^63-1}]. Note that for numbers outside this range the way that Java * truncates integral values may yield unexpected results.
*

{@code null} values

* The FoundationDB tuple specification has a special type-code for {@code None}; {@code nil}; or, * as Java would understand it, {@code null}. * The behavior of the layer in the presence of {@code null} varies by type with the intention - * of matching expected behavior in Java. {@code byte[]} and {@link String}s can be {@code null}, - * where integral numbers (i.e. {@code long}s) cannot. - * This means that the typed getters ({@link #getBytes(int) getBytes()} and {@link #getString(int) getString()}) + * of matching expected behavior in Java. {@code byte[]}, {@link String}s, {@link UUID}s, and + * nested {@link List}s and {@code Tuple}s can be {@code null}, + * whereas numbers (e.g., {@code long}s and {@code double}s) and booleans cannot. + * This means that the typed getters ({@link #getBytes(int) getBytes()}, {@link #getString(int) getString()}), + * {@link #getUUID(int) getUUID()}, {@link #getNestedTuple(int) getNestedTuple()}, and {@link #getNestedList(int) getNestedList()}) * will return {@code null} if the entry at that location was {@code null} and the typed adds * ({@link #add(byte[])} and {@link #add(String)}) will accept {@code null}. The - * {@link #getLong(int) typed get for integers}, however, will throw a {@code NullPointerException} if + * {@link #getLong(int) typed get for integers} and other typed getters, however, will throw a {@link NullPointerException} if * the entry in the {@code Tuple} was {@code null} at that position.
*
* This class is not thread safe. */ public class Tuple implements Comparable, Iterable { + private static IterableComparator comparator = new IterableComparator(); + private List elements; private Tuple(List elements, Object newItem) { @@ -72,12 +83,12 @@ public class Tuple implements Comparable, Iterable { /** * Creates a copy of this {@code Tuple} with an appended last element. The parameter - * is untyped but only {@link String}, {@code byte[]}, {@link Number}s, and {@code null} are allowed. - * All {@code Number}s are converted to a 8 byte integral value, so all floating point - * information is lost. + * is untyped but only {@link String}, {@code byte[]}, {@link Number}s, {@link UUID}s, + * {@link Boolean}s, {@link List}s, {@code Tuple}s, and {@code null} are allowed. If an object of + * another type is passed, then an {@link IllegalArgumentException} is thrown. * * @param o the object to append. Must be {@link String}, {@code byte[]}, - * {@link Number}s, or {@code null}. + * {@link Number}s, {@link UUID}, {@link List}, {@link Boolean}, or {@code null}. * * @return a newly created {@code Tuple} */ @@ -85,6 +96,10 @@ public class Tuple implements Comparable, Iterable { if(o != null && !(o instanceof String) && !(o instanceof byte[]) && + !(o instanceof UUID) && + !(o instanceof List) && + !(o instanceof Tuple) && + !(o instanceof Boolean) && !(o instanceof Number)) { throw new IllegalArgumentException("Parameter type (" + o.getClass().getName() + ") not recognized"); } @@ -124,6 +139,92 @@ public class Tuple implements Comparable, Iterable { return new Tuple(this.elements, b); } + /** + * Creates a copy of this {@code Tuple} with a {@code boolean} appended as the last element. + * + * @param b the {@code boolean} to append + * + * @return a newly created {@code Tuple} + */ + public Tuple add(boolean b) { + return new Tuple(this.elements, b); + } + + /** + * Creates a copy of this {@code Tuple} with a {@link UUID} appended as the last element. + * + * @param uuid the {@link UUID} to append + * + * @return a newly created {@code Tuple} + */ + public Tuple add(UUID uuid) { + return new Tuple(this.elements, uuid); + } + + /** + * Creates a copy of this {@code Tuple} with a {@link BigInteger} appended as the last element. + * As {@link Tuple}s cannot contain {@code null} numeric types, a {@link NullPointerException} + * is raised if a {@code null} argument is passed. + * + * @param bi the {@link BigInteger} to append + * + * @return a newly created {@code Tuple} + */ + public Tuple add(BigInteger bi) { + if(bi == null) { + throw new NullPointerException("Number types in Tuple cannot be null"); + } + return new Tuple(this.elements, bi); + } + + /** + * Creates a copy of this {@code Tuple} with a {@code float} appended as the last element. + * + * @param f the {@code float} to append + * + * @return a newly created {@code Tuple} + */ + public Tuple add(float f) { + return new Tuple(this.elements, f); + } + + /** + * Creates a copy of this {@code Tuple} with a {@code double} appended as the last element. + * + * @param d the {@code double} to append + * + * @return a newly created {@code Tuple} + */ + public Tuple add(double d) { + return new Tuple(this.elements, d); + } + + /** + * Creates a copy of this {@code Tuple} with an {@link List} appended as the last element. + * This does not add the elements individually (for that, use {@link Tuple#addAll(List) Tuple.addAll}). + * This adds the list as a single elemented nested within the outer {@code Tuple}. + * + * @param l the {@link List} to append + * + * @return a newly created {@code Tuple} + */ + public Tuple add(List l) { + return new Tuple(this.elements, l); + } + + /** + * Creates a copy of this {@code Tuple} with a {@code Tuple} appended as the last element. + * This does not add the elements individually (for that, use {@link Tuple#addAll(Tuple) Tuple.addAll}). + * This adds the list as a single elemented nested within the outer {@code Tuple}. + * + * @param t the {@code Tuple} to append + * + * @return a newly created {@code Tuple} + */ + public Tuple add(Tuple t) { + return new Tuple(this.elements, t); + } + /** * Creates a copy of this {@code Tuple} with a {@code byte} array appended as the last element. * @@ -320,6 +421,138 @@ public class Tuple implements Comparable, Iterable { return (String)o; } + /** + * Gets an indexed item as a {@link BigInteger}. This function will not do type conversion + * and so will throw a {@code ClassCastException} if the tuple element is not of + * a {@code Number} type. If the underlying type is a floating point value, this + * will lead to a loss of precision. The element at the index may not be {@code null}. + * + * @param index the location of the element to return + * + * @return the item at {@code index} as a {@link BigInteger} + */ + public BigInteger getBigInteger(int index) { + Object o = this.elements.get(index); + if(o == null) + throw new NullPointerException("Number types in Tuples may not be null"); + if(o instanceof BigInteger) { + return (BigInteger)o; + } else { + return BigInteger.valueOf(((Number)o).longValue()); + } + } + + /** + * Gets an indexed item as a {@code float}. This function will not do type conversion + * and so will throw a {@code ClassCastException} if the element is not a number type. + * The element at the index may not be {@code null}. + * + * @param index the location of the item to return + * + * @return the item at {@code index} as a {@code float} + */ + public float getFloat(int index) { + Object o = this.elements.get(index); + if(o == null) + throw new NullPointerException("Number types in Tuples may not be null"); + return ((Number)o).floatValue(); + } + + /** + * Gets an indexed item as a {@code double}. This function will not do type conversion + * and so will throw a {@code ClassCastException} if the element is not a number type. + * The element at the index may not be {@code null}. + * + * @param index the location of the item to return + * + * @return the item at {@code index} as a {@code double} + */ + public double getDouble(int index) { + Object o = this.elements.get(index); + if(o == null) + throw new NullPointerException("Number types in Tuples may not be null"); + return ((Number)o).doubleValue(); + } + + /** + * Gets an indexed item as a {@code boolean}. This function will not do type conversion + * and so will throw a {@code ClassCastException} if the element is not a {@code Boolean}. + * The element at the index may not be {@code null}. + * + * @param index the location of the item to return + * + * @return the item at {@code index} as a {@code boolean} + */ + public boolean getBoolean(int index) { + Object o = this.elements.get(index); + if(o == null) + throw new NullPointerException("Boolean type in Tuples may not be null"); + return (Boolean)o; + + } + + /** + * Gets an indexed item as a {@link UUID}. This function will not do type conversion + * and so will throw a {@code ClassCastException} if the element is not a {@code UUID}. + * The element at the index may not be {@code null}. + * + * @param index the location of the item to return + * + * @return the item at {@code index} as a {@link UUID} + */ + public UUID getUUID(int index) { + Object o = this.elements.get(index); + if(o == null) + return null; + return (UUID)o; + } + + /** + * Gets an indexed item as a {@link List}. This function will not do type conversion + * and so will throw a {@code ClassCastException} if the element is not a {@link List} + * or {@code Tuple}. The element at the index may be {@code null}. + * + * @param index the location of the item to return + * + * @return the item at {@code index} as a {@link List} + */ + public List getNestedList(int index) { + Object o = this.elements.get(index); + if(o == null) { + return null; + } else if(o instanceof Tuple) { + return ((Tuple)o).getItems(); + } else if(o instanceof List) { + List ret = new LinkedList(); + ret.addAll((List)o); + return ret; + } else { + throw new ClassCastException("Cannot convert item of type " + o.getClass() + " to list"); + } + } + + /** + * Gets an indexed item as a {@link Tuple}. This function will not do type conversion + * and so will throw a {@code ClassCastException} if the element is not a {@link List} + * or {@code Tuple}. The element at the index may be {@code null}. + * + * @param index the location of the item to return + * + * @return the item at {@code index} as a {@link List} + */ + public Tuple getNestedTuple(int index) { + Object o = this.elements.get(index); + if(o == null) { + return null; + } else if(o instanceof Tuple) { + return (Tuple)o; + } else if(o instanceof List) { + return Tuple.fromItems((List)o); + } else { + throw new ClassCastException("Cannot convert item of type " + o.getClass() + " to tuple"); + } + } + /** * Gets an indexed item without forcing a type. * @@ -400,7 +633,7 @@ public class Tuple implements Comparable, Iterable { */ @Override public int compareTo(Tuple t) { - return ByteArrayUtil.compareUnsigned(this.pack(), t.pack()); + return comparator.compare(elements, t.elements); } /** @@ -473,7 +706,8 @@ public class Tuple implements Comparable, Iterable { /** * Creates a new {@code Tuple} from a variable number of elements. The elements * must follow the type guidelines from {@link Tuple#addObject(Object) add}, and so - * can only be {@link String}s, {@code byte[]}s, {@link Number}s, or {@code null}s. + * can only be {@link String}s, {@code byte[]}s, {@link Number}s, {@link UUID}s, + * {@link Boolean}s, {@link List}s, {@code Tuple}s, or {@code null}s. * * @param items the elements from which to create the {@code Tuple}. * @@ -490,7 +724,8 @@ public class Tuple implements Comparable, Iterable { /** * Efficiently creates a new {@code Tuple} from a list of objects. The elements * must follow the type guidelines from {@link Tuple#addObject(Object) add}, and so - * can only be {@link String}s, {@code byte[]}s, {@link Number}s, or {@code null}s. + * can only be {@link String}s, {@code byte[]}s, {@link Number}s, {@link UUID}s, + * {@link Boolean}s, {@link List}s, {@code Tuple}s, or {@code null}s. * * @param items the elements from which to create the {@code Tuple}. * @@ -518,7 +753,8 @@ public class Tuple implements Comparable, Iterable { /** * Creates a new {@code Tuple} from a variable number of elements. The elements * must follow the type guidelines from {@link Tuple#addObject(Object) add}, and so - * can only be {@link String}s, {@code byte[]}s, {@link Number}s, or {@code null}s. + * can only be {@link String}s, {@code byte[]}s, {@link Number}s, {@link UUID}s, + * {@link Boolean}s, {@link List}s, {@code Tuple}s, or {@code null}s. * * @param items the elements from which to create the {@code Tuple}. * @@ -544,13 +780,27 @@ public class Tuple implements Comparable, Iterable { t = t.add(Long.MIN_VALUE + 1); t = t.add(Long.MIN_VALUE); t = t.add("foo"); + t = t.addObject(null); + t = t.add(false); + t = t.add(true); + t = t.add(3.14159); + t = t.add(3.14159f); + t = t.add(java.util.UUID.randomUUID()); + t = t.add(t.getItems()); + t = t.add(t); + t = t.add(new BigInteger("100000000000000000000000000000000000000000000")); + t = t.add(new BigInteger("-100000000000000000000000000000000000000000000")); byte[] bytes = t.pack(); System.out.println("Packed: " + ByteArrayUtil.printable(bytes)); List items = Tuple.fromBytes(bytes).getItems(); for(Object obj : items) { - System.out.println(" -> type: (" + obj.getClass().getName() + "): " + obj); + if (obj != null) + System.out.println(" -> type: (" + obj.getClass().getName() + "): " + obj); + else + System.out.println(" -> type: (null): null"); } + t = Tuple.fromStream(t.stream().map(item -> { if(item instanceof String) { return ((String)item).toUpperCase(); @@ -559,6 +809,33 @@ public class Tuple implements Comparable, Iterable { } })); System.out.println("Upper cased: " + t); + + Tuple t2 = Tuple.fromBytes(bytes); + System.out.println("t2.getLong(0): " + t2.getLong(0)); + System.out.println("t2.getBigInteger(1): " + t2.getBigInteger(1)); + System.out.println("t2.getString(9): " + t2.getString(9)); + System.out.println("t2.get(10): " + t2.get(10)); + System.out.println("t2.getBoolean(11): " + t2.getBoolean(11)); + System.out.println("t2.getBoolean(12): " + t2.getBoolean(12)); + System.out.println("t2.getDouble(13): " + t2.getDouble(13)); + System.out.println("t2.getFloat(13): " + t2.getFloat(13)); + System.out.println("t2.getLong(13): " + t2.getLong(13)); + System.out.println("t2.getBigInteger(13): " + t2.getBigInteger(13)); + System.out.println("t2.getDouble(14): " + t2.getDouble(14)); + System.out.println("t2.getFloat(14): " + t2.getFloat(14)); + System.out.println("t2.getLong(14): " + t2.getLong(14)); + System.out.println("t2.getBigInteger(14): " + t2.getBigInteger(14)); + System.out.println("t2.getNestedList(17): " + t2.getNestedList(17)); + System.out.println("t2.getNestedTuple(17): " + t2.getNestedTuple(17)); + + System.out.println("(2*(Long.MAX_VALUE+1),) = " + ByteArrayUtil.printable(Tuple.from( + BigInteger.valueOf(Long.MAX_VALUE).add(BigInteger.ONE).shiftLeft(1) + ).pack())); + System.out.println("(2*Long.MIN_VALUE,) = " + ByteArrayUtil.printable(Tuple.from( + BigInteger.valueOf(Long.MIN_VALUE).multiply(new BigInteger("2")) + ).pack())); + System.out.println("2*Long.MIN_VALUE = " + Tuple.fromBytes(Tuple.from( + BigInteger.valueOf(Long.MIN_VALUE).multiply(new BigInteger("2"))).pack()).getBigInteger(0)); } private static Tuple createTuple(int items) { diff --git a/bindings/java/src-completable/main/com/apple/cie/foundationdb/tuple/TupleUtil.java b/bindings/java/src-completable/main/com/apple/cie/foundationdb/tuple/TupleUtil.java index 6cdd93d2be..af611e0fd6 100644 --- a/bindings/java/src-completable/main/com/apple/cie/foundationdb/tuple/TupleUtil.java +++ b/bindings/java/src-completable/main/com/apple/cie/foundationdb/tuple/TupleUtil.java @@ -28,12 +28,26 @@ import java.util.ArrayList; import java.util.Arrays; import java.util.LinkedList; import java.util.List; +import java.util.UUID; class TupleUtil { - private static final byte nil = 0x0; + private static final byte nil = 0x00; private static final byte[] nil_rep = new byte[] {nil, (byte)0xFF}; private static final BigInteger[] size_limits; private static final Charset UTF8; + private static final IterableComparator iterableComparator; + + private static final byte BYTES_CODE = 0x01; + private static final byte STRING_CODE = 0x02; + private static final byte NESTED_CODE = 0x05; + private static final byte INT_ZERO_CODE = 0x14; + private static final byte POS_INT_END = 0x1d; + private static final byte NEG_INT_START = 0x0b; + private static final byte FLOAT_CODE = 0x20; + private static final byte DOUBLE_CODE = 0x21; + private static final byte FALSE_CODE = 0x26; + private static final byte TRUE_CODE = 0x27; + private static final byte UUID_CODE = 0x30; static { size_limits = new BigInteger[9]; @@ -41,6 +55,7 @@ class TupleUtil { size_limits[i] = (BigInteger.ONE).shiftLeft(i * 8).subtract(BigInteger.ONE); } UTF8 = Charset.forName("UTF-8"); + iterableComparator = new IterableComparator(); } static class DecodeResult { @@ -53,27 +68,109 @@ class TupleUtil { } } + static int byteLength(byte[] bytes) { + for(int i = 0; i < bytes.length; i++) { + if(bytes[i] == 0x00) continue; + return bytes.length - i; + } + return 0; + } + + /** + * Takes the Big-Endian byte representation of a floating point number and adjusts + * it so that it sorts correctly. For encoding, if the sign bit is 1 (the number + * is negative), then we need to flip all of the bits; otherwise, just flip the + * sign bit. For decoding, if the sign bit is 0 (the number is negative), then + * we also need to flip all of the bits; otherwise, just flip the sign bit. + * This will mutate in place the given array. + * + * @param bytes Big-Endian IEEE encoding of a floating point number + * @param start the (zero-indexed) first byte in the array to mutate + * @param encode true if we encoding the float and false if we are decoding + * @return the encoded {@code byte[]} + */ + static byte[] floatingPointCoding(byte[] bytes, int start, boolean encode) { + if(encode && (bytes[start] & (byte)0x80) != (byte)0x00) { + for(int i = start; i < bytes.length; i++) { + bytes[i] = (byte) (bytes[i] ^ 0xff); + } + } else if(!encode && (bytes[start] & (byte)0x80) != (byte)0x80) { + for(int i = start; i < bytes.length; i++) { + bytes[i] = (byte) (bytes[i] ^ 0xff); + } + } else { + bytes[start] = (byte) (0x80 ^ bytes[start]); + } + + return bytes; + } + public static byte[] join(List items) { return ByteArrayUtil.join(null, items); } - static byte[] encode(Object t) { + static int getCodeFor(Object o) { + if(o == null) + return nil; + if(o instanceof byte[]) + return BYTES_CODE; + if(o instanceof String) + return STRING_CODE; + if(o instanceof Float) + return FLOAT_CODE; + if(o instanceof Double) + return DOUBLE_CODE; + if(o instanceof Boolean) + return FALSE_CODE; + if(o instanceof UUID) + return UUID_CODE; + if(o instanceof Number) + return INT_ZERO_CODE; + if(o instanceof List) + return NESTED_CODE; + if(o instanceof Tuple) + return NESTED_CODE; + throw new IllegalArgumentException("Unsupported data type: " + o.getClass().getName()); + } + + static byte[] encode(Object t, boolean nested) { if(t == null) - return new byte[] {nil}; + if (nested) + return new byte[]{nil, (byte) 0xff}; + else + return new byte[]{nil}; if(t instanceof byte[]) return encode((byte[])t); if(t instanceof String) return encode((String)t); + if(t instanceof BigInteger) + return encode((BigInteger)t); + if(t instanceof Float) + return encode((Float)t); + if(t instanceof Double) + return encode((Double)t); + if(t instanceof Boolean) + return encode((Boolean)t); + if(t instanceof UUID) + return encode((UUID)t); if(t instanceof Number) return encode(((Number)t).longValue()); + if(t instanceof List) + return encode((List)t); + if(t instanceof Tuple) + return encode(((Tuple)t).getItems()); throw new IllegalArgumentException("Unsupported data type: " + t.getClass().getName()); } + static byte[] encode(Object t) { + return encode(t, false); + } + static byte[] encode(byte[] bytes) { List list = new ArrayList(3); - list.add(new byte[] {0x1}); - list.add(ByteArrayUtil.replace(bytes, new byte[] {0x0}, nil_rep)); - list.add(new byte[] {0x0}); + list.add(new byte[] {BYTES_CODE}); + list.add(ByteArrayUtil.replace(bytes, new byte[] {nil}, nil_rep)); + list.add(new byte[] {nil}); //System.out.println("Joining bytes..."); return ByteArrayUtil.join(null, list); @@ -81,39 +178,68 @@ class TupleUtil { static byte[] encode(String s) { List list = new ArrayList(3); - list.add(new byte[] {0x2}); - list.add(ByteArrayUtil.replace(s.getBytes(UTF8), new byte[] {0x0}, nil_rep)); - list.add(new byte[] {0x0}); + list.add(new byte[] {STRING_CODE}); + list.add(ByteArrayUtil.replace(s.getBytes(UTF8), new byte[] {nil}, nil_rep)); + list.add(new byte[] {nil}); //System.out.println("Joining string..."); return ByteArrayUtil.join(null, list); } - static byte[] encode(long i) { + static byte[] encode(BigInteger i) { //System.out.println("Encoding integral " + i); - if(i == 0) { - return new byte[] { 20 }; + if(i.equals(BigInteger.ZERO)) { + return new byte[] { INT_ZERO_CODE }; } - if(i > 0) { - int n = ByteArrayUtil.bisectLeft(size_limits, BigInteger.valueOf(i)); + byte[] bytes = i.toByteArray(); + if(i.compareTo(BigInteger.ZERO) > 0) { + if(i.compareTo(size_limits[size_limits.length-1]) > 0) { + int length = byteLength(bytes); + if(length > 0xff) { + throw new IllegalArgumentException("BigInteger magnitude is too large (more than 255 bytes)"); + } + byte[] result = new byte[length + 2]; + result[0] = POS_INT_END; + result[1] = (byte)(length); + System.arraycopy(bytes, bytes.length - length, result, 2, length); + return result; + } + int n = ByteArrayUtil.bisectLeft(size_limits, i); assert n <= size_limits.length; - byte[] bytes = ByteBuffer.allocate(8).order(ByteOrder.BIG_ENDIAN).putLong(i).array(); + //byte[] bytes = ByteBuffer.allocate(8).order(ByteOrder.BIG_ENDIAN).putLong(i).array(); //System.out.println(" -- integral has 'n' of " + n + " and output bytes of " + bytes.length); byte[] result = new byte[n+1]; - result[0] = (byte)(20 + n); + result[0] = (byte)(INT_ZERO_CODE + n); System.arraycopy(bytes, bytes.length - n, result, 1, n); return result; } - BigInteger bI = BigInteger.valueOf(i); - int n = ByteArrayUtil.bisectLeft(size_limits, bI.negate()); + if(i.negate().compareTo(size_limits[size_limits.length-1]) > 0) { + int length = byteLength(i.negate().toByteArray()); + if(length > 0xff) { + throw new IllegalArgumentException("BigInteger magnitude is too large (more than 255 bytes)"); + } + BigInteger offset = BigInteger.ONE.shiftLeft(length*8).subtract(BigInteger.ONE); + byte[] adjusted = i.add(offset).toByteArray(); + byte[] result = new byte[length + 2]; + result[0] = NEG_INT_START; + result[1] = (byte)(length ^ 0xff); + if(adjusted.length >= length) { + System.arraycopy(adjusted, adjusted.length - length, result, 2, length); + } else { + Arrays.fill(result, 2, result.length - adjusted.length, (byte)0x00); + System.arraycopy(adjusted, 0, result, result.length - adjusted.length, adjusted.length); + } + return result; + } + int n = ByteArrayUtil.bisectLeft(size_limits, i.negate()); assert n >= 0 && n < size_limits.length; // can we do this? it seems to be required for the following statement - long maxv = size_limits[n].add(bI).longValue(); - byte[] bytes = ByteBuffer.allocate(8).order(ByteOrder.BIG_ENDIAN).putLong(maxv).array(); + long maxv = size_limits[n].add(i).longValue(); + byte[] adjustedBytes = ByteBuffer.allocate(8).order(ByteOrder.BIG_ENDIAN).putLong(maxv).array(); byte[] result = new byte[n+1]; result[0] = (byte)(20 - n); - System.arraycopy(bytes, bytes.length - n, result, 1, n); + System.arraycopy(adjustedBytes, adjustedBytes.length - n, result, 1, n); return result; } @@ -121,23 +247,63 @@ class TupleUtil { return encode(i.longValue()); } + static byte[] encode(long i) { + return encode(BigInteger.valueOf(i)); + } + + static byte[] encode(Float f) { + byte[] result = ByteBuffer.allocate(5).order(ByteOrder.BIG_ENDIAN).put(FLOAT_CODE).putFloat(f).array(); + floatingPointCoding(result, 1, true); + return result; + } + + static byte[] encode(Double d) { + byte[] result = ByteBuffer.allocate(9).order(ByteOrder.BIG_ENDIAN).put(DOUBLE_CODE).putDouble(d).array(); + floatingPointCoding(result, 1, true); + return result; + } + + static byte[] encode(Boolean b) { + if (b) { + return new byte[] {TRUE_CODE}; + } else { + return new byte[] {FALSE_CODE}; + } + } + + static byte[] encode(UUID uuid) { + return ByteBuffer.allocate(17).put(UUID_CODE).order(ByteOrder.BIG_ENDIAN) + .putLong(uuid.getMostSignificantBits()).putLong(uuid.getLeastSignificantBits()) + .array(); + } + + static byte[] encode(List value) { + List parts = new LinkedList(); + parts.add(new byte[]{NESTED_CODE}); + for(Object t : value) { + parts.add(encode(t, true)); + } + parts.add(new byte[]{0x00}); + return ByteArrayUtil.join(null, parts); + } + static DecodeResult decode(byte[] rep, int pos, int last) { //System.out.println("Decoding '" + ArrayUtils.printable(rep) + "' at " + pos); // SOMEDAY: codes over 127 will be a problem with the signed Java byte mess int code = rep[pos]; int start = pos + 1; - if(code == 0x0) { + if(code == nil) { return new DecodeResult(start, null); } - if(code == 0x1) { + if(code == BYTES_CODE) { int end = ByteArrayUtil.findTerminator(rep, (byte)0x0, (byte)0xff, start, last); //System.out.println("End of byte string: " + end); byte[] range = ByteArrayUtil.replace(rep, start, end - start, nil_rep, new byte[] { nil }); //System.out.println(" -> byte string contents: '" + ArrayUtils.printable(range) + "'"); return new DecodeResult(end + 1, range); } - if(code == 0x2) { + if(code == STRING_CODE) { int end = ByteArrayUtil.findTerminator(rep, (byte)0x0, (byte)0xff, start, last); //System.out.println("End of UTF8 string: " + end); byte[] stringBytes = ByteArrayUtil.replace(rep, start, end - start, nil_rep, new byte[] { nil }); @@ -145,11 +311,44 @@ class TupleUtil { //System.out.println(" -> UTF8 string contents: '" + str + "'"); return new DecodeResult(end + 1, str); } - if(code >=12 && code <=28) { + if(code == FLOAT_CODE) { + byte[] resBytes = Arrays.copyOfRange(rep, start, start+4); + floatingPointCoding(resBytes, 0, false); + float res = ByteBuffer.wrap(resBytes).order(ByteOrder.BIG_ENDIAN).getFloat(); + return new DecodeResult(start + 4, res); + } + if(code == DOUBLE_CODE) { + byte[] resBytes = Arrays.copyOfRange(rep, start, start+8); + floatingPointCoding(resBytes, 0, false); + double res = ByteBuffer.wrap(resBytes).order(ByteOrder.BIG_ENDIAN).getDouble(); + return new DecodeResult(start + 8, res); + } + if(code == FALSE_CODE) { + return new DecodeResult(start, false); + } + if(code == TRUE_CODE) { + return new DecodeResult(start, true); + } + if(code == UUID_CODE) { + ByteBuffer bb = ByteBuffer.wrap(rep, start, 16).order(ByteOrder.BIG_ENDIAN); + long msb = bb.getLong(); + long lsb = bb.getLong(); + return new DecodeResult(start + 16, new UUID(msb, lsb)); + } + if(code == POS_INT_END) { + int n = rep[start] & 0xff; + return new DecodeResult(start + n + 1, new BigInteger(ByteArrayUtil.join(new byte[]{0x00}, Arrays.copyOfRange(rep, start+1, start+n+1)))); + } + if(code == NEG_INT_START) { + int n = (rep[start] ^ 0xff) & 0xff; + BigInteger origValue = new BigInteger(ByteArrayUtil.join(new byte[]{0x00}, Arrays.copyOfRange(rep, start+1, start+n+1))); + BigInteger offset = BigInteger.ONE.shiftLeft(n*8).subtract(BigInteger.ONE); + return new DecodeResult(start + n + 1, origValue.subtract(offset)); + } + if(code > NEG_INT_START && code < POS_INT_END) { // decode a long byte[] longBytes = new byte[9]; - Arrays.fill(longBytes, (byte)0); - boolean upper = code >= 20; + boolean upper = code >= INT_ZERO_CODE; int n = upper ? code - 20 : 20 - code; int end = start + n; @@ -157,23 +356,108 @@ class TupleUtil { throw new RuntimeException("Invalid tuple (possible truncation)"); } - System.arraycopy(rep, start, longBytes, 9-n, n); + System.arraycopy(rep, start, longBytes, longBytes.length-n, n); if (!upper) - for(int i=9-n; i<9; i++) - longBytes[i] = (byte)~longBytes[i]; + for(int i=longBytes.length-n; i0) - throw new RuntimeException("Value out of range for type long."); - + // Convert to long if in range -- otherwise, leave as BigInteger. + if (val.compareTo(BigInteger.valueOf(Long.MIN_VALUE))<0|| + val.compareTo(BigInteger.valueOf(Long.MAX_VALUE))>0) { + // This can occur if the thing can be represented with 8 bytes but not + // the right sign information. + return new DecodeResult(end, val); + } return new DecodeResult(end, val.longValue()); } + if(code == NESTED_CODE) { + List items = new LinkedList(); + int endPos = start; + while(endPos < rep.length) { + if(rep[endPos] == nil) { + if(endPos + 1 < rep.length && rep[endPos+1] == (byte)0xff) { + items.add(null); + endPos += 2; + } else { + endPos += 1; + break; + } + } else { + DecodeResult subResult = decode(rep, endPos, last); + items.add(subResult.o); + endPos = subResult.end; + } + } + return new DecodeResult(endPos, items); + } throw new IllegalArgumentException("Unknown tuple data type " + code + " at index " + pos); } + static int compareItems(Object item1, Object item2) { + int code1 = TupleUtil.getCodeFor(item1); + int code2 = TupleUtil.getCodeFor(item2); + + if(code1 != code2) { + return Integer.compare(code1, code2); + } + + if(code1 == nil) { + // All null's are equal. (Some may be more equal than others.) + return 0; + } + if(code1 == BYTES_CODE) { + return ByteArrayUtil.compareUnsigned((byte[])item1, (byte[])item2); + } + if(code1 == STRING_CODE) { + return ByteArrayUtil.compareUnsigned(((String)item1).getBytes(UTF8), ((String)item2).getBytes(UTF8)); + } + if(code1 == INT_ZERO_CODE) { + BigInteger bi1; + if(item1 instanceof BigInteger) { + bi1 = (BigInteger)item1; + } else { + bi1 = BigInteger.valueOf(((Number)item1).longValue()); + } + BigInteger bi2; + if(item2 instanceof BigInteger) { + bi2 = (BigInteger)item2; + } else { + bi2 = BigInteger.valueOf(((Number)item2).longValue()); + } + return bi1.compareTo(bi2); + } + if(code1 == DOUBLE_CODE) { + // This is done over vanilla double comparison basically to handle NaN + // sorting correctly. + byte[] encoded1 = encode((Double)item1); + byte[] encoded2 = encode((Double)item2); + return ByteArrayUtil.compareUnsigned(encoded1, encoded2); + } + if(code1 == FLOAT_CODE) { + // This is done for the same reason that double comparison is done + // that way. + byte[] encoded1 = encode((Float)item1); + byte[] encoded2 = encode((Float)item2); + return ByteArrayUtil.compareUnsigned(encoded1, encoded2); + } + if(code1 == FALSE_CODE) { + return Boolean.compare((Boolean)item1, (Boolean)item2); + } + if(code1 == UUID_CODE) { + // Java UUID.compareTo is signed. + byte[] encoded1 = encode((UUID)item1); + byte[] encoded2 = encode((UUID)item2); + return ByteArrayUtil.compareUnsigned(encoded1, encoded2); + } + if(code1 == NESTED_CODE) { + return iterableComparator.compare((Iterable)item1, (Iterable)item2); + } + throw new IllegalArgumentException("Unknown tuple data type: " + item1.getClass()); + } + static List unpack(byte[] bytes, int start, int length) { List items = new LinkedList(); int pos = start; @@ -188,7 +472,7 @@ class TupleUtil { static byte[] pack(List items) { if(items.size() == 0) - return new byte[0]; + return new byte[0]; List parts = new ArrayList(items.size()); for(Object t : items) { diff --git a/bindings/java/src-completable/test/com/apple/cie/foundationdb/test/AsyncStackTester.java b/bindings/java/src-completable/test/com/apple/cie/foundationdb/test/AsyncStackTester.java index 508ee2a334..8b013d6bcd 100644 --- a/bindings/java/src-completable/test/com/apple/cie/foundationdb/test/AsyncStackTester.java +++ b/bindings/java/src-completable/test/com/apple/cie/foundationdb/test/AsyncStackTester.java @@ -20,8 +20,11 @@ package com.apple.cie.foundationdb.test; +import java.math.BigInteger; import java.util.*; +import java.nio.ByteBuffer; +import java.nio.ByteOrder; import java.util.concurrent.CompletableFuture; import java.util.function.Function; @@ -435,8 +438,9 @@ public class AsyncStackTester { return inst.popParams(2).thenApplyAsync(new Function, Void>() { @Override public Void apply(List params) { - long result = StackUtils.getNumber(params.get(0)).longValue() - - StackUtils.getNumber(params.get(1)).longValue(); + BigInteger result = StackUtils.getBigInteger(params.get(0)).subtract( + StackUtils.getBigInteger(params.get(1)) + ); inst.push(result); return null; } @@ -508,6 +512,70 @@ public class AsyncStackTester { } }); } + else if(op == StackOperation.TUPLE_SORT) { + return inst.popParam().thenComposeAsync(new Function>() { + @Override + public CompletableFuture apply(Object param) { + final int listSize = StackUtils.getInt(param); + return inst.popParams(listSize).thenApply(new Function, Void>() { + @Override + public Void apply(List rawElements) { + List tuples = new ArrayList(listSize); + for(Object o : rawElements) { + tuples.add(Tuple.fromBytes((byte[])o)); + } + Collections.sort(tuples); + for(Tuple t : tuples) { + inst.push(t.pack()); + } + return null; + } + }); + } + }); + } + else if (op == StackOperation.ENCODE_FLOAT) { + return inst.popParam().thenApply(new Function() { + @Override + public Void apply(Object param) { + byte[] fBytes = (byte[])param; + float value = ByteBuffer.wrap(fBytes).order(ByteOrder.BIG_ENDIAN).getFloat(); + inst.push(value); + return null; + } + }); + } + else if (op == StackOperation.ENCODE_DOUBLE) { + return inst.popParam().thenApply(new Function() { + @Override + public Void apply(Object param) { + byte[] dBytes = (byte[])param; + double value = ByteBuffer.wrap(dBytes).order(ByteOrder.BIG_ENDIAN).getDouble(); + inst.push(value); + return null; + } + }); + } + else if (op == StackOperation.DECODE_FLOAT) { + return inst.popParam().thenApply(new Function() { + @Override + public Void apply(Object param) { + float value = ((Number)param).floatValue(); + inst.push(ByteBuffer.allocate(4).order(ByteOrder.BIG_ENDIAN).putFloat(value).array()); + return null; + } + }); + } + else if (op == StackOperation.DECODE_DOUBLE) { + return inst.popParam().thenApply(new Function() { + @Override + public Void apply(Object param) { + double value = ((Number)param).doubleValue(); + inst.push(ByteBuffer.allocate(8).order(ByteOrder.BIG_ENDIAN).putDouble(value).array()); + return null; + } + }); + } else if(op == StackOperation.UNIT_TESTS) { inst.context.db.options().setLocationCacheSize(100001); return inst.context.db.runAsync(tr -> { diff --git a/bindings/java/src-completable/test/com/apple/cie/foundationdb/test/StackOperation.java b/bindings/java/src-completable/test/com/apple/cie/foundationdb/test/StackOperation.java index 10c8c5c253..fc81d9104a 100644 --- a/bindings/java/src-completable/test/com/apple/cie/foundationdb/test/StackOperation.java +++ b/bindings/java/src-completable/test/com/apple/cie/foundationdb/test/StackOperation.java @@ -62,6 +62,11 @@ enum StackOperation { TUPLE_PACK, TUPLE_UNPACK, TUPLE_RANGE, + TUPLE_SORT, + ENCODE_FLOAT, + ENCODE_DOUBLE, + DECODE_FLOAT, + DECODE_DOUBLE, UNIT_TESTS, /* Possibly unimplemented */ LOG_STACK diff --git a/bindings/java/src-completable/test/com/apple/cie/foundationdb/test/StackTester.java b/bindings/java/src-completable/test/com/apple/cie/foundationdb/test/StackTester.java index 458e617789..6f1d289a61 100644 --- a/bindings/java/src-completable/test/com/apple/cie/foundationdb/test/StackTester.java +++ b/bindings/java/src-completable/test/com/apple/cie/foundationdb/test/StackTester.java @@ -20,8 +20,11 @@ package com.apple.cie.foundationdb.test; +import java.math.BigInteger; import java.util.*; +import java.nio.ByteBuffer; +import java.nio.ByteOrder; import java.util.concurrent.CompletableFuture; import java.util.concurrent.CompletionException; import java.util.function.Function; @@ -312,8 +315,9 @@ public class StackTester { } else if(op == StackOperation.SUB) { List params = inst.popParams(2).join(); - long result = StackUtils.getNumber(params.get(0)).longValue() - StackUtils.getNumber(params.get(1)).longValue(); - inst.push(result); + BigInteger a = StackUtils.getBigInteger(params.get(0)); + BigInteger b = StackUtils.getBigInteger(params.get(1)); + inst.push(a.subtract(b)); } else if(op == StackOperation.CONCAT) { List params = inst.popParams(2).join(); @@ -352,6 +356,40 @@ public class StackTester { inst.push(range.begin); inst.push(range.end); } + else if (op == StackOperation.TUPLE_SORT) { + int listSize = StackUtils.getInt(inst.popParam().join()); + List rawElements = inst.popParams(listSize).join(); + List tuples = new ArrayList(listSize); + for(Object o : rawElements) { + tuples.add(Tuple.fromBytes((byte[])o)); + } + Collections.sort(tuples); + for(Tuple t : tuples) { + inst.push(t.pack()); + } + } + else if (op == StackOperation.ENCODE_FLOAT) { + Object param = inst.popParam().join(); + byte[] fBytes = (byte[])param; + float value = ByteBuffer.wrap(fBytes).order(ByteOrder.BIG_ENDIAN).getFloat(); + inst.push(value); + } + else if (op == StackOperation.ENCODE_DOUBLE) { + Object param = inst.popParam().join(); + byte[] dBytes = (byte[])param; + double value = ByteBuffer.wrap(dBytes).order(ByteOrder.BIG_ENDIAN).getDouble(); + inst.push(value); + } + else if (op == StackOperation.DECODE_FLOAT) { + Object param = inst.popParam().join(); + float value = ((Number)param).floatValue(); + inst.push(ByteBuffer.allocate(4).order(ByteOrder.BIG_ENDIAN).putFloat(value).array()); + } + else if (op == StackOperation.DECODE_DOUBLE) { + Object param = inst.popParam().join(); + double value = ((Number)param).doubleValue(); + inst.push(ByteBuffer.allocate(8).order(ByteOrder.BIG_ENDIAN).putDouble(value).array()); + } else if(op == StackOperation.UNIT_TESTS) { try { inst.context.db.options().setLocationCacheSize(100001); diff --git a/bindings/java/src-completable/test/com/apple/cie/foundationdb/test/StackUtils.java b/bindings/java/src-completable/test/com/apple/cie/foundationdb/test/StackUtils.java index 3d0a6e50bc..502653b71a 100644 --- a/bindings/java/src-completable/test/com/apple/cie/foundationdb/test/StackUtils.java +++ b/bindings/java/src-completable/test/com/apple/cie/foundationdb/test/StackUtils.java @@ -20,6 +20,7 @@ package com.apple.cie.foundationdb.test; +import java.math.BigInteger; import java.util.List; import java.util.concurrent.CompletableFuture; import java.util.concurrent.CompletionException; @@ -92,6 +93,14 @@ public class StackUtils { return ((Number)object); } + static BigInteger getBigInteger(Object object) { + if (object instanceof BigInteger) { + return (BigInteger)object; + } else { + return BigInteger.valueOf(((Number)object).longValue()); + } + } + static boolean getBoolean(Object o) { return getBoolean(o, null); } diff --git a/bindings/java/src/main/com/apple/cie/foundationdb/tuple/IterableComparator.java b/bindings/java/src/main/com/apple/cie/foundationdb/tuple/IterableComparator.java new file mode 100644 index 0000000000..d09bf976de --- /dev/null +++ b/bindings/java/src/main/com/apple/cie/foundationdb/tuple/IterableComparator.java @@ -0,0 +1,107 @@ +/* + * IterableComparator.java + * + * This source file is part of the FoundationDB open source project + * + * Copyright 2013-2018 Apple Inc. and the FoundationDB project authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.apple.cie.foundationdb.tuple; + +import java.util.Comparator; +import java.util.Iterator; + +/** + * A {@link Tuple}-compatible {@link Comparator} that will sort {@link Iterable}s + * in a manner that is consistent with the byte-representation of {@link Tuple}s. + * In particular, if one has two {@link Tuple}s, {@code tuple1} and {@code tuple2}, + * it is the case that: + * + * 
+ * {@code
+ *    tuple1.compareTo(tuple2)
+ *      == new IterableComparator().compare(tuple1, tuple2)
+ *      == new IterableComparator().compare(tuple1.getItems(), tuple2.getItems()),
+ *      == ByteArrayUtil.compareUnsigned(tuple1.pack(), tuple2.pack())}
+ *
+ * + *

+ * The individual elements of the {@link Iterable} must be of a type that can + * be serialized by a {@link Tuple}. For items of identical types, they will be + * sorted in a way that is consistent with their natural ordering with a few + * caveats: + *

+ * + *
    + *
  • + * For floating point types, negative NaN values are sorted before all regular values, and + * positive NaN values are sorted after all regular values. + *
    • + *
  • + * Single-precision floating point numbers are sorted before + * all double-precision floating point numbers. + *
    • + *
  • + * UUIDs are sorted by their unsigned Big-Endian byte representation + * rather than their signed byte representation (which is the behavior of + * {@link java.util.UUID#compareTo(java.util.UUID) UUID.compareTo()}). + *
    • + *
  • Strings are sorted explicitly by their UTF-8 byte representation
    • + *
  • Nested {@link Tuple}s and {@link java.util.List List}s are sorted element-wise.
    • + *
+ */ +public class IterableComparator implements Comparator> { + /** + * Creates a new {@code IterableComparator}. This {@link Comparator} has + * no internal state. + */ + public IterableComparator() {} + + /** + * Compare two {@link Iterable}s in a way consistent with their + * byte representation. This is done element-wise and is consistent + * with a number of other ways of sorting {@link Tuple}s. This will + * raise an {@link IllegalArgumentException} if any of the items + * of either {@link Iterable} cannot be serialized by a {@link Tuple}. + * + * @param iterable1 the first {@link Iterable} of items + * @param iterable2 the second {@link Iterable} of items + * @return a negative number if the first iterable would sort before the second + * when serialized, a positive number if the opposite is true, and zero + * if the two are equal + */ + @Override + public int compare(Iterable iterable1, Iterable iterable2) { + Iterator i1 = iterable1.iterator(); + Iterator i2 = iterable2.iterator(); + + while(i1.hasNext() && i2.hasNext()) { + int itemComp = TupleUtil.compareItems(i1.next(), i2.next()); + if(itemComp != 0) { + return itemComp; + } + } + + if(i1.hasNext()) { + // iterable2 is a prefix of iterable1. + return 1; + } + if(i2.hasNext()) { + // iterable1 is a prefix of iterable2. + return -1; + } + return 0; + } +} diff --git a/bindings/java/src/main/com/apple/cie/foundationdb/tuple/Tuple.java b/bindings/java/src/main/com/apple/cie/foundationdb/tuple/Tuple.java index 5601285ba4..64d4b58aad 100644 --- a/bindings/java/src/main/com/apple/cie/foundationdb/tuple/Tuple.java +++ b/bindings/java/src/main/com/apple/cie/foundationdb/tuple/Tuple.java @@ -20,12 +20,15 @@ package com.apple.cie.foundationdb.tuple; +import java.math.BigInteger; import java.util.ArrayList; import java.util.Arrays; +import java.util.Collection; import java.util.Collections; import java.util.Iterator; import java.util.LinkedList; import java.util.List; +import java.util.UUID; import com.apple.cie.foundationdb.Range; @@ -37,26 +40,34 @@ import com.apple.cie.foundationdb.Range; * ideal for building a variety of higher-level data models.
*

Types

* A {@code Tuple} can - * contain byte arrays ({@code byte[]}), {@link String}s, {@link Number}s, and {@code null}. All - * {@code Number}s will be converted to a {@code long} integral value, so all - * floating point information will be lost and their range will be constrained to the range - * [{@code 2^63-1}, {@code -2^63}]. Note that for numbers outside this range the way that Java + * contain byte arrays ({@code byte[]}), {@link String}s, {@link Number}s, {@link UUID}s, + * {@code boolean}s, {@link List}s, other {@code Tuple}s, and {@code null}. + * {@link Float} and {@link Double} instances will be serialized as single- and double-precision + * numbers respectively, and {@link BigInteger}s within the range [{@code -2^2040+1}, + * {@code 2^2040-1}] are serialized without loss of precision (those outside the range + * will raise an {@link IllegalArgumentException}). All other {@code Number}s will be converted to + * a {@code long} integral value, so the range will be constrained to + * [{@code -2^63}, {@code 2^63-1}]. Note that for numbers outside this range the way that Java * truncates integral values may yield unexpected results.
*

{@code null} values

* The FoundationDB tuple specification has a special type-code for {@code None}; {@code nil}; or, * as Java would understand it, {@code null}. * The behavior of the layer in the presence of {@code null} varies by type with the intention - * of matching expected behavior in Java. {@code byte[]} and {@link String}s can be {@code null}, - * where integral numbers (i.e. {@code long}s) cannot. - * This means that the typed getters ({@link #getBytes(int) getBytes()} and {@link #getString(int) getString()}) + * of matching expected behavior in Java. {@code byte[]}, {@link String}s, {@link UUID}s, and + * nested {@link List}s and {@code Tuple}s can be {@code null}, + * whereas numbers (e.g., {@code long}s and {@code double}s) and booleans cannot. + * This means that the typed getters ({@link #getBytes(int) getBytes()}, {@link #getString(int) getString()}), + * {@link #getUUID(int) getUUID()}, {@link #getNestedTuple(int) getNestedTuple()}, and {@link #getNestedList(int) getNestedList()}) * will return {@code null} if the entry at that location was {@code null} and the typed adds * ({@link #add(byte[])} and {@link #add(String)}) will accept {@code null}. The - * {@link #getLong(int) typed get for integers}, however, will throw a {@code NullPointerException} if + * {@link #getLong(int) typed get for integers} and other typed getters, however, will throw a {@link NullPointerException} if * the entry in the {@code Tuple} was {@code null} at that position.
*
* This class is not thread safe. */ public class Tuple implements Comparable, Iterable { + private static IterableComparator comparator = new IterableComparator(); + private List elements; private Tuple(List elements, Object newItem) { @@ -70,12 +81,12 @@ public class Tuple implements Comparable, Iterable { /** * Creates a copy of this {@code Tuple} with an appended last element. The parameter - * is untyped but only {@link String}, {@code byte[]}, {@link Number}s, and {@code null} are allowed. - * All {@code Number}s are converted to a 8 byte integral value, so all floating point - * information is lost. + * is untyped but only {@link String}, {@code byte[]}, {@link Number}s, {@link UUID}s, + * {@link Boolean}s, {@link List}s, {@code Tuple}s, and {@code null} are allowed. If an object of + * another type is passed, then an {@link IllegalArgumentException} is thrown. * * @param o the object to append. Must be {@link String}, {@code byte[]}, - * {@link Number}s, or {@code null}. + * {@link Number}s, {@link UUID}, {@link List}, {@link Boolean}, or {@code null}. * * @return a newly created {@code Tuple} */ @@ -83,6 +94,10 @@ public class Tuple implements Comparable, Iterable { if(o != null && !(o instanceof String) && !(o instanceof byte[]) && + !(o instanceof UUID) && + !(o instanceof List) && + !(o instanceof Tuple) && + !(o instanceof Boolean) && !(o instanceof Number)) { throw new IllegalArgumentException("Parameter type (" + o.getClass().getName() + ") not recognized"); } @@ -122,6 +137,92 @@ public class Tuple implements Comparable, Iterable { return new Tuple(this.elements, b); } + /** + * Creates a copy of this {@code Tuple} with a {@code boolean} appended as the last element. + * + * @param b the {@code boolean} to append + * + * @return a newly created {@code Tuple} + */ + public Tuple add(boolean b) { + return new Tuple(this.elements, b); + } + + /** + * Creates a copy of this {@code Tuple} with a {@link UUID} appended as the last element. + * + * @param uuid the {@link UUID} to append + * + * @return a newly created {@code Tuple} + */ + public Tuple add(UUID uuid) { + return new Tuple(this.elements, uuid); + } + + /** + * Creates a copy of this {@code Tuple} with a {@link BigInteger} appended as the last element. + * As {@link Tuple}s cannot contain {@code null} numeric types, a {@link NullPointerException} + * is raised if a {@code null} argument is passed. + * + * @param bi the {@link BigInteger} to append + * + * @return a newly created {@code Tuple} + */ + public Tuple add(BigInteger bi) { + if(bi == null) { + throw new NullPointerException("Number types in Tuple cannot be null"); + } + return new Tuple(this.elements, bi); + } + + /** + * Creates a copy of this {@code Tuple} with a {@code float} appended as the last element. + * + * @param f the {@code float} to append + * + * @return a newly created {@code Tuple} + */ + public Tuple add(float f) { + return new Tuple(this.elements, f); + } + + /** + * Creates a copy of this {@code Tuple} with a {@code double} appended as the last element. + * + * @param d the {@code double} to append + * + * @return a newly created {@code Tuple} + */ + public Tuple add(double d) { + return new Tuple(this.elements, d); + } + + /** + * Creates a copy of this {@code Tuple} with an {@link List} appended as the last element. + * This does not add the elements individually (for that, use {@link Tuple#addAll(List) Tuple.addAll}). + * This adds the list as a single elemented nested within the outer {@code Tuple}. + * + * @param l the {@link List} to append + * + * @return a newly created {@code Tuple} + */ + public Tuple add(List l) { + return new Tuple(this.elements, l); + } + + /** + * Creates a copy of this {@code Tuple} with a {@code Tuple} appended as the last element. + * This does not add the elements individually (for that, use {@link Tuple#addAll(Tuple) Tuple.addAll}). + * This adds the list as a single elemented nested within the outer {@code Tuple}. + * + * @param t the {@code Tuple} to append + * + * @return a newly created {@code Tuple} + */ + public Tuple add(Tuple t) { + return new Tuple(this.elements, t); + } + /** * Creates a copy of this {@code Tuple} with a {@code byte} array appended as the last element. * @@ -309,6 +410,138 @@ public class Tuple implements Comparable, Iterable { return (String)o; } + /** + * Gets an indexed item as a {@link BigInteger}. This function will not do type conversion + * and so will throw a {@code ClassCastException} if the tuple element is not of + * a {@code Number} type. If the underlying type is a floating point value, this + * will lead to a loss of precision. The element at the index may not be {@code null}. + * + * @param index the location of the element to return + * + * @return the item at {@code index} as a {@link BigInteger} + */ + public BigInteger getBigInteger(int index) { + Object o = this.elements.get(index); + if(o == null) + throw new NullPointerException("Number types in Tuples may not be null"); + if(o instanceof BigInteger) { + return (BigInteger)o; + } else { + return BigInteger.valueOf(((Number)o).longValue()); + } + } + + /** + * Gets an indexed item as a {@code float}. This function will not do type conversion + * and so will throw a {@code ClassCastException} if the element is not a number type. + * The element at the index may not be {@code null}. + * + * @param index the location of the item to return + * + * @return the item at {@code index} as a {@code float} + */ + public float getFloat(int index) { + Object o = this.elements.get(index); + if(o == null) + throw new NullPointerException("Number types in Tuples may not be null"); + return ((Number)o).floatValue(); + } + + /** + * Gets an indexed item as a {@code double}. This function will not do type conversion + * and so will throw a {@code ClassCastException} if the element is not a number type. + * The element at the index may not be {@code null}. + * + * @param index the location of the item to return + * + * @return the item at {@code index} as a {@code double} + */ + public double getDouble(int index) { + Object o = this.elements.get(index); + if(o == null) + throw new NullPointerException("Number types in Tuples may not be null"); + return ((Number)o).doubleValue(); + } + + /** + * Gets an indexed item as a {@code boolean}. This function will not do type conversion + * and so will throw a {@code ClassCastException} if the element is not a {@code Boolean}. + * The element at the index may not be {@code null}. + * + * @param index the location of the item to return + * + * @return the item at {@code index} as a {@code boolean} + */ + public boolean getBoolean(int index) { + Object o = this.elements.get(index); + if(o == null) + throw new NullPointerException("Boolean type in Tuples may not be null"); + return (Boolean)o; + + } + + /** + * Gets an indexed item as a {@link UUID}. This function will not do type conversion + * and so will throw a {@code ClassCastException} if the element is not a {@code UUID}. + * The element at the index may not be {@code null}. + * + * @param index the location of the item to return + * + * @return the item at {@code index} as a {@link UUID} + */ + public UUID getUUID(int index) { + Object o = this.elements.get(index); + if(o == null) + return null; + return (UUID)o; + } + + /** + * Gets an indexed item as a {@link List}. This function will not do type conversion + * and so will throw a {@code ClassCastException} if the element is not a {@link List} + * or {@code Tuple}. The element at the index may be {@code null}. + * + * @param index the location of the item to return + * + * @return the item at {@code index} as a {@link List} + */ + public List getNestedList(int index) { + Object o = this.elements.get(index); + if(o == null) { + return null; + } else if(o instanceof Tuple) { + return ((Tuple)o).getItems(); + } else if(o instanceof List) { + List ret = new LinkedList(); + ret.addAll((List)o); + return ret; + } else { + throw new ClassCastException("Cannot convert item of type " + o.getClass() + " to list"); + } + } + + /** + * Gets an indexed item as a {@link Tuple}. This function will not do type conversion + * and so will throw a {@code ClassCastException} if the element is not a {@link List} + * or {@code Tuple}. The element at the index may be {@code null}. + * + * @param index the location of the item to return + * + * @return the item at {@code index} as a {@link List} + */ + public Tuple getNestedTuple(int index) { + Object o = this.elements.get(index); + if(o == null) { + return null; + } else if(o instanceof Tuple) { + return (Tuple)o; + } else if(o instanceof List) { + return Tuple.fromItems((List)o); + } else { + throw new ClassCastException("Cannot convert item of type " + o.getClass() + " to tuple"); + } + } + /** * Gets an indexed item without forcing a type. * @@ -389,7 +622,7 @@ public class Tuple implements Comparable, Iterable { */ @Override public int compareTo(Tuple t) { - return ByteArrayUtil.compareUnsigned(this.pack(), t.pack()); + return comparator.compare(elements, t.elements); } /** @@ -462,7 +695,8 @@ public class Tuple implements Comparable, Iterable { /** * Creates a new {@code Tuple} from a variable number of elements. The elements * must follow the type guidelines from {@link Tuple#addObject(Object) add}, and so - * can only be {@link String}s, {@code byte[]}s, {@link Number}s, or {@code null}s. + * can only be {@link String}s, {@code byte[]}s, {@link Number}s, {@link UUID}s, + * {@link Boolean}s, {@link List}s, {@code Tuple}s, or {@code null}s. * * @param items the elements from which to create the {@code Tuple}. * @@ -479,7 +713,8 @@ public class Tuple implements Comparable, Iterable { /** * Efficiently creates a new {@code Tuple} from a list of objects. The elements * must follow the type guidelines from {@link Tuple#addObject(Object) add}, and so - * can only be {@link String}s, {@code byte[]}s, {@link Number}s, or {@code null}s. + * can only be {@link String}s, {@code byte[]}s, {@link Number}s, {@link UUID}s, + * {@link Boolean}s, {@link List}s, {@code Tuple}s, or {@code null}s. * * @param items the elements from which to create the {@code Tuple}. * @@ -492,7 +727,8 @@ public class Tuple implements Comparable, Iterable { /** * Creates a new {@code Tuple} from a variable number of elements. The elements * must follow the type guidelines from {@link Tuple#addObject(Object) add}, and so - * can only be {@link String}s, {@code byte[]}s, {@link Number}s, or {@code null}s. + * can only be {@link String}s, {@code byte[]}s, {@link Number}s, {@link UUID}s, + * {@link Boolean}s, {@link List}s, {@code Tuple}s, or {@code null}s. * * @param items the elements from which to create the {@code Tuple}. * @@ -518,12 +754,51 @@ public class Tuple implements Comparable, Iterable { t = t.add(Long.MIN_VALUE + 1); t = t.add(Long.MIN_VALUE); t = t.add("foo"); + t = t.addObject(null); + t = t.add(false); + t = t.add(true); + t = t.add(3.14159); + t = t.add(3.14159f); + t = t.add(java.util.UUID.randomUUID()); + t = t.add(t.getItems()); + t = t.add(t); + t = t.add(new BigInteger("100000000000000000000000000000000000000000000")); + t = t.add(new BigInteger("-100000000000000000000000000000000000000000000")); byte[] bytes = t.pack(); System.out.println("Packed: " + ByteArrayUtil.printable(bytes)); List items = Tuple.fromBytes(bytes).getItems(); for(Object obj : items) { - System.out.println(" -> type: (" + obj.getClass().getName() + "): " + obj); + if (obj != null) + System.out.println(" -> type: (" + obj.getClass().getName() + "): " + obj); + else + System.out.println(" -> type: (null): null"); } + Tuple t2 = Tuple.fromBytes(bytes); + System.out.println("t2.getLong(0): " + t2.getLong(0)); + System.out.println("t2.getBigInteger(1): " + t2.getBigInteger(1)); + System.out.println("t2.getString(9): " + t2.getString(9)); + System.out.println("t2.get(10): " + t2.get(10)); + System.out.println("t2.getBoolean(11): " + t2.getBoolean(11)); + System.out.println("t2.getBoolean(12): " + t2.getBoolean(12)); + System.out.println("t2.getDouble(13): " + t2.getDouble(13)); + System.out.println("t2.getFloat(13): " + t2.getFloat(13)); + System.out.println("t2.getLong(13): " + t2.getLong(13)); + System.out.println("t2.getBigInteger(13): " + t2.getBigInteger(13)); + System.out.println("t2.getDouble(14): " + t2.getDouble(14)); + System.out.println("t2.getFloat(14): " + t2.getFloat(14)); + System.out.println("t2.getLong(14): " + t2.getLong(14)); + System.out.println("t2.getBigInteger(14): " + t2.getBigInteger(14)); + System.out.println("t2.getNestedList(17): " + t2.getNestedList(17)); + System.out.println("t2.getNestedTuple(17): " + t2.getNestedTuple(17)); + + System.out.println("(2*(Long.MAX_VALUE+1),) = " + ByteArrayUtil.printable(Tuple.from( + BigInteger.valueOf(Long.MAX_VALUE).add(BigInteger.ONE).shiftLeft(1) + ).pack())); + System.out.println("(2*Long.MIN_VALUE,) = " + ByteArrayUtil.printable(Tuple.from( + BigInteger.valueOf(Long.MIN_VALUE).multiply(new BigInteger("2")) + ).pack())); + System.out.println("2*Long.MIN_VALUE = " + Tuple.fromBytes(Tuple.from( + BigInteger.valueOf(Long.MIN_VALUE).multiply(new BigInteger("2"))).pack()).getBigInteger(0)); } private static Tuple createTuple(int items) { diff --git a/bindings/java/src/main/com/apple/cie/foundationdb/tuple/TupleUtil.java b/bindings/java/src/main/com/apple/cie/foundationdb/tuple/TupleUtil.java index 6cdd93d2be..94d2ae3d0b 100644 --- a/bindings/java/src/main/com/apple/cie/foundationdb/tuple/TupleUtil.java +++ b/bindings/java/src/main/com/apple/cie/foundationdb/tuple/TupleUtil.java @@ -28,12 +28,26 @@ import java.util.ArrayList; import java.util.Arrays; import java.util.LinkedList; import java.util.List; +import java.util.UUID; class TupleUtil { - private static final byte nil = 0x0; + private static final byte nil = 0x00; private static final byte[] nil_rep = new byte[] {nil, (byte)0xFF}; private static final BigInteger[] size_limits; private static final Charset UTF8; + private static final IterableComparator iterableComparator; + + private static final byte BYTES_CODE = 0x01; + private static final byte STRING_CODE = 0x02; + private static final byte NESTED_CODE = 0x05; + private static final byte INT_ZERO_CODE = 0x14; + private static final byte POS_INT_END = 0x1d; + private static final byte NEG_INT_START = 0x0b; + private static final byte FLOAT_CODE = 0x20; + private static final byte DOUBLE_CODE = 0x21; + private static final byte FALSE_CODE = 0x26; + private static final byte TRUE_CODE = 0x27; + private static final byte UUID_CODE = 0x30; static { size_limits = new BigInteger[9]; @@ -41,6 +55,7 @@ class TupleUtil { size_limits[i] = (BigInteger.ONE).shiftLeft(i * 8).subtract(BigInteger.ONE); } UTF8 = Charset.forName("UTF-8"); + iterableComparator = new IterableComparator(); } static class DecodeResult { @@ -53,27 +68,110 @@ class TupleUtil { } } + static int byteLength(byte[] bytes) { + for(int i = 0; i < bytes.length; i++) { + if(bytes[i] == 0x00) continue; + return bytes.length - i; + } + return 0; + } + + /** + * Takes the Big-Endian byte representation of a floating point number and adjusts + * it so that it sorts correctly. For encoding, if the sign bit is 1 (the number + * is negative), then we need to flip all of the bits; otherwise, just flip the + * sign bit. For decoding, if the sign bit is 0 (the number is negative), then + * we also need to flip all of the bits; otherwise, just flip the sign bit. + * This will mutate in place the given array. + * + * @param bytes Big-Endian IEEE encoding of a floating point number + * @param start the (zero-indexed) first byte in the array to mutate + * @param encode true if we encoding the float and false if we are decoding + * @return the encoded {@code byte[]} + */ + static byte[] floatingPointCoding(byte[] bytes, int start, boolean encode) { + if(encode && (bytes[start] & (byte)0x80) != (byte)0x00) { + for(int i = start; i < bytes.length; i++) { + bytes[i] = (byte) (bytes[i] ^ 0xff); + } + } else if(!encode && (bytes[start] & (byte)0x80) != (byte)0x80) { + for(int i = start; i < bytes.length; i++) { + bytes[i] = (byte) (bytes[i] ^ 0xff); + } + } else { + bytes[start] = (byte) (0x80 ^ bytes[start]); + } + + return bytes; + } + + public static byte[] join(List items) { return ByteArrayUtil.join(null, items); } - static byte[] encode(Object t) { + static int getCodeFor(Object o) { + if(o == null) + return nil; + if(o instanceof byte[]) + return BYTES_CODE; + if(o instanceof String) + return STRING_CODE; + if(o instanceof Float) + return FLOAT_CODE; + if(o instanceof Double) + return DOUBLE_CODE; + if(o instanceof Boolean) + return FALSE_CODE; + if(o instanceof UUID) + return UUID_CODE; + if(o instanceof Number) + return INT_ZERO_CODE; + if(o instanceof List) + return NESTED_CODE; + if(o instanceof Tuple) + return NESTED_CODE; + throw new IllegalArgumentException("Unsupported data type: " + o.getClass().getName()); + } + + static byte[] encode(Object t, boolean nested) { if(t == null) - return new byte[] {nil}; + if (nested) + return new byte[]{nil, (byte) 0xff}; + else + return new byte[]{nil}; if(t instanceof byte[]) return encode((byte[])t); if(t instanceof String) return encode((String)t); + if(t instanceof BigInteger) + return encode((BigInteger)t); + if(t instanceof Float) + return encode((Float)t); + if(t instanceof Double) + return encode((Double)t); + if(t instanceof Boolean) + return encode((Boolean)t); + if(t instanceof UUID) + return encode((UUID)t); if(t instanceof Number) return encode(((Number)t).longValue()); + if(t instanceof List) + return encode((List)t); + if(t instanceof Tuple) + return encode(((Tuple)t).getItems()); throw new IllegalArgumentException("Unsupported data type: " + t.getClass().getName()); } + static byte[] encode(Object t) { + return encode(t, false); + } + static byte[] encode(byte[] bytes) { List list = new ArrayList(3); - list.add(new byte[] {0x1}); - list.add(ByteArrayUtil.replace(bytes, new byte[] {0x0}, nil_rep)); - list.add(new byte[] {0x0}); + list.add(new byte[] {BYTES_CODE}); + list.add(ByteArrayUtil.replace(bytes, new byte[] {nil}, nil_rep)); + list.add(new byte[] {nil}); //System.out.println("Joining bytes..."); return ByteArrayUtil.join(null, list); @@ -81,39 +179,68 @@ class TupleUtil { static byte[] encode(String s) { List list = new ArrayList(3); - list.add(new byte[] {0x2}); - list.add(ByteArrayUtil.replace(s.getBytes(UTF8), new byte[] {0x0}, nil_rep)); - list.add(new byte[] {0x0}); + list.add(new byte[] {STRING_CODE}); + list.add(ByteArrayUtil.replace(s.getBytes(UTF8), new byte[] {nil}, nil_rep)); + list.add(new byte[] {nil}); //System.out.println("Joining string..."); return ByteArrayUtil.join(null, list); } - static byte[] encode(long i) { + static byte[] encode(BigInteger i) { //System.out.println("Encoding integral " + i); - if(i == 0) { - return new byte[] { 20 }; + if(i.equals(BigInteger.ZERO)) { + return new byte[] { INT_ZERO_CODE }; } - if(i > 0) { - int n = ByteArrayUtil.bisectLeft(size_limits, BigInteger.valueOf(i)); + byte[] bytes = i.toByteArray(); + if(i.compareTo(BigInteger.ZERO) > 0) { + if(i.compareTo(size_limits[size_limits.length-1]) > 0) { + int length = byteLength(bytes); + if(length > 0xff) { + throw new IllegalArgumentException("BigInteger magnitude is too large (more than 255 bytes)"); + } + byte[] result = new byte[length + 2]; + result[0] = POS_INT_END; + result[1] = (byte)(length); + System.arraycopy(bytes, bytes.length - length, result, 2, length); + return result; + } + int n = ByteArrayUtil.bisectLeft(size_limits, i); assert n <= size_limits.length; - byte[] bytes = ByteBuffer.allocate(8).order(ByteOrder.BIG_ENDIAN).putLong(i).array(); + //byte[] bytes = ByteBuffer.allocate(8).order(ByteOrder.BIG_ENDIAN).putLong(i).array(); //System.out.println(" -- integral has 'n' of " + n + " and output bytes of " + bytes.length); byte[] result = new byte[n+1]; - result[0] = (byte)(20 + n); + result[0] = (byte)(INT_ZERO_CODE + n); System.arraycopy(bytes, bytes.length - n, result, 1, n); return result; } - BigInteger bI = BigInteger.valueOf(i); - int n = ByteArrayUtil.bisectLeft(size_limits, bI.negate()); + if(i.negate().compareTo(size_limits[size_limits.length-1]) > 0) { + int length = byteLength(i.negate().toByteArray()); + if(length > 0xff) { + throw new IllegalArgumentException("BigInteger magnitude is too large (more than 255 bytes)"); + } + BigInteger offset = BigInteger.ONE.shiftLeft(length*8).subtract(BigInteger.ONE); + byte[] adjusted = i.add(offset).toByteArray(); + byte[] result = new byte[length + 2]; + result[0] = NEG_INT_START; + result[1] = (byte)(length ^ 0xff); + if(adjusted.length >= length) { + System.arraycopy(adjusted, adjusted.length - length, result, 2, length); + } else { + Arrays.fill(result, 2, result.length - adjusted.length, (byte)0x00); + System.arraycopy(adjusted, 0, result, result.length - adjusted.length, adjusted.length); + } + return result; + } + int n = ByteArrayUtil.bisectLeft(size_limits, i.negate()); assert n >= 0 && n < size_limits.length; // can we do this? it seems to be required for the following statement - long maxv = size_limits[n].add(bI).longValue(); - byte[] bytes = ByteBuffer.allocate(8).order(ByteOrder.BIG_ENDIAN).putLong(maxv).array(); + long maxv = size_limits[n].add(i).longValue(); + byte[] adjustedBytes = ByteBuffer.allocate(8).order(ByteOrder.BIG_ENDIAN).putLong(maxv).array(); byte[] result = new byte[n+1]; result[0] = (byte)(20 - n); - System.arraycopy(bytes, bytes.length - n, result, 1, n); + System.arraycopy(adjustedBytes, adjustedBytes.length - n, result, 1, n); return result; } @@ -121,23 +248,63 @@ class TupleUtil { return encode(i.longValue()); } + static byte[] encode(long i) { + return encode(BigInteger.valueOf(i)); + } + + static byte[] encode(Float f) { + byte[] result = ByteBuffer.allocate(5).order(ByteOrder.BIG_ENDIAN).put(FLOAT_CODE).putFloat(f).array(); + floatingPointCoding(result, 1, true); + return result; + } + + static byte[] encode(Double d) { + byte[] result = ByteBuffer.allocate(9).order(ByteOrder.BIG_ENDIAN).put(DOUBLE_CODE).putDouble(d).array(); + floatingPointCoding(result, 1, true); + return result; + } + + static byte[] encode(Boolean b) { + if (b) { + return new byte[] {TRUE_CODE}; + } else { + return new byte[] {FALSE_CODE}; + } + } + + static byte[] encode(UUID uuid) { + return ByteBuffer.allocate(17).put(UUID_CODE).order(ByteOrder.BIG_ENDIAN) + .putLong(uuid.getMostSignificantBits()).putLong(uuid.getLeastSignificantBits()) + .array(); + } + + static byte[] encode(List value) { + List parts = new LinkedList(); + parts.add(new byte[]{NESTED_CODE}); + for(Object t : value) { + parts.add(encode(t, true)); + } + parts.add(new byte[]{0x00}); + return ByteArrayUtil.join(null, parts); + } + static DecodeResult decode(byte[] rep, int pos, int last) { //System.out.println("Decoding '" + ArrayUtils.printable(rep) + "' at " + pos); // SOMEDAY: codes over 127 will be a problem with the signed Java byte mess int code = rep[pos]; int start = pos + 1; - if(code == 0x0) { + if(code == nil) { return new DecodeResult(start, null); } - if(code == 0x1) { + if(code == BYTES_CODE) { int end = ByteArrayUtil.findTerminator(rep, (byte)0x0, (byte)0xff, start, last); //System.out.println("End of byte string: " + end); byte[] range = ByteArrayUtil.replace(rep, start, end - start, nil_rep, new byte[] { nil }); //System.out.println(" -> byte string contents: '" + ArrayUtils.printable(range) + "'"); return new DecodeResult(end + 1, range); } - if(code == 0x2) { + if(code == STRING_CODE) { int end = ByteArrayUtil.findTerminator(rep, (byte)0x0, (byte)0xff, start, last); //System.out.println("End of UTF8 string: " + end); byte[] stringBytes = ByteArrayUtil.replace(rep, start, end - start, nil_rep, new byte[] { nil }); @@ -145,11 +312,44 @@ class TupleUtil { //System.out.println(" -> UTF8 string contents: '" + str + "'"); return new DecodeResult(end + 1, str); } - if(code >=12 && code <=28) { + if(code == FLOAT_CODE) { + byte[] resBytes = Arrays.copyOfRange(rep, start, start+4); + floatingPointCoding(resBytes, 0, false); + float res = ByteBuffer.wrap(resBytes).order(ByteOrder.BIG_ENDIAN).getFloat(); + return new DecodeResult(start + 4, res); + } + if(code == DOUBLE_CODE) { + byte[] resBytes = Arrays.copyOfRange(rep, start, start+8); + floatingPointCoding(resBytes, 0, false); + double res = ByteBuffer.wrap(resBytes).order(ByteOrder.BIG_ENDIAN).getDouble(); + return new DecodeResult(start + 8, res); + } + if(code == FALSE_CODE) { + return new DecodeResult(start, false); + } + if(code == TRUE_CODE) { + return new DecodeResult(start, true); + } + if(code == UUID_CODE) { + ByteBuffer bb = ByteBuffer.wrap(rep, start, 16).order(ByteOrder.BIG_ENDIAN); + long msb = bb.getLong(); + long lsb = bb.getLong(); + return new DecodeResult(start + 16, new UUID(msb, lsb)); + } + if(code == POS_INT_END) { + int n = rep[start] & 0xff; + return new DecodeResult(start + n + 1, new BigInteger(ByteArrayUtil.join(new byte[]{0x00}, Arrays.copyOfRange(rep, start+1, start+n+1)))); + } + if(code == NEG_INT_START) { + int n = (rep[start] ^ 0xff) & 0xff; + BigInteger origValue = new BigInteger(ByteArrayUtil.join(new byte[]{0x00}, Arrays.copyOfRange(rep, start+1, start+n+1))); + BigInteger offset = BigInteger.ONE.shiftLeft(n*8).subtract(BigInteger.ONE); + return new DecodeResult(start + n + 1, origValue.subtract(offset)); + } + if(code > NEG_INT_START && code < POS_INT_END) { // decode a long byte[] longBytes = new byte[9]; - Arrays.fill(longBytes, (byte)0); - boolean upper = code >= 20; + boolean upper = code >= INT_ZERO_CODE; int n = upper ? code - 20 : 20 - code; int end = start + n; @@ -157,23 +357,108 @@ class TupleUtil { throw new RuntimeException("Invalid tuple (possible truncation)"); } - System.arraycopy(rep, start, longBytes, 9-n, n); + System.arraycopy(rep, start, longBytes, longBytes.length-n, n); if (!upper) - for(int i=9-n; i<9; i++) - longBytes[i] = (byte)~longBytes[i]; + for(int i=longBytes.length-n; i0) - throw new RuntimeException("Value out of range for type long."); - + // Convert to long if in range -- otherwise, leave as BigInteger. + if (val.compareTo(BigInteger.valueOf(Long.MIN_VALUE))<0|| + val.compareTo(BigInteger.valueOf(Long.MAX_VALUE))>0) { + // This can occur if the thing can be represented with 8 bytes but not + // the right sign information. + return new DecodeResult(end, val); + } return new DecodeResult(end, val.longValue()); } + if(code == NESTED_CODE) { + List items = new LinkedList(); + int endPos = start; + while(endPos < rep.length) { + if(rep[endPos] == nil) { + if(endPos + 1 < rep.length && rep[endPos+1] == (byte)0xff) { + items.add(null); + endPos += 2; + } else { + endPos += 1; + break; + } + } else { + DecodeResult subResult = decode(rep, endPos, last); + items.add(subResult.o); + endPos = subResult.end; + } + } + return new DecodeResult(endPos, items); + } throw new IllegalArgumentException("Unknown tuple data type " + code + " at index " + pos); } + static int compareItems(Object item1, Object item2) { + int code1 = TupleUtil.getCodeFor(item1); + int code2 = TupleUtil.getCodeFor(item2); + + if(code1 != code2) { + return Integer.compare(code1, code2); + } + + if(code1 == nil) { + // All null's are equal. (Some may be more equal than others.) + return 0; + } + if(code1 == BYTES_CODE) { + return ByteArrayUtil.compareUnsigned((byte[])item1, (byte[])item2); + } + if(code1 == STRING_CODE) { + return ByteArrayUtil.compareUnsigned(((String)item1).getBytes(UTF8), ((String)item2).getBytes(UTF8)); + } + if(code1 == INT_ZERO_CODE) { + BigInteger bi1; + if(item1 instanceof BigInteger) { + bi1 = (BigInteger)item1; + } else { + bi1 = BigInteger.valueOf(((Number)item1).longValue()); + } + BigInteger bi2; + if(item2 instanceof BigInteger) { + bi2 = (BigInteger)item2; + } else { + bi2 = BigInteger.valueOf(((Number)item2).longValue()); + } + return bi1.compareTo(bi2); + } + if(code1 == DOUBLE_CODE) { + // This is done over vanilla double comparison basically to handle NaN + // sorting correctly. + byte[] encoded1 = encode((Double)item1); + byte[] encoded2 = encode((Double)item2); + return ByteArrayUtil.compareUnsigned(encoded1, encoded2); + } + if(code1 == FLOAT_CODE) { + // This is done for the same reason that double comparison is done + // that way. + byte[] encoded1 = encode((Float)item1); + byte[] encoded2 = encode((Float)item2); + return ByteArrayUtil.compareUnsigned(encoded1, encoded2); + } + if(code1 == FALSE_CODE) { + return Boolean.compare((Boolean)item1, (Boolean)item2); + } + if(code1 == UUID_CODE) { + // Java UUID.compareTo is signed. + byte[] encoded1 = encode((UUID)item1); + byte[] encoded2 = encode((UUID)item2); + return ByteArrayUtil.compareUnsigned(encoded1, encoded2); + } + if(code1 == NESTED_CODE) { + return iterableComparator.compare((Iterable)item1, (Iterable)item2); + } + throw new IllegalArgumentException("Unknown tuple data type: " + item1.getClass()); + } + static List unpack(byte[] bytes, int start, int length) { List items = new LinkedList(); int pos = start; @@ -188,7 +473,7 @@ class TupleUtil { static byte[] pack(List items) { if(items.size() == 0) - return new byte[0]; + return new byte[0]; List parts = new ArrayList(items.size()); for(Object t : items) { diff --git a/bindings/java/src/test/com/apple/cie/foundationdb/test/AsyncStackTester.java b/bindings/java/src/test/com/apple/cie/foundationdb/test/AsyncStackTester.java index f38c3e56a0..e7143fba09 100644 --- a/bindings/java/src/test/com/apple/cie/foundationdb/test/AsyncStackTester.java +++ b/bindings/java/src/test/com/apple/cie/foundationdb/test/AsyncStackTester.java @@ -20,6 +20,9 @@ package com.apple.cie.foundationdb.test; +import java.math.BigInteger; +import java.nio.ByteBuffer; +import java.nio.ByteOrder; import java.util.*; import com.apple.cie.foundationdb.Cluster; @@ -471,8 +474,9 @@ public class AsyncStackTester { return inst.popParams(2).flatMap(new Function, Future>() { @Override public Future apply(List params) { - long result = StackUtils.getNumber(params.get(0)).longValue() - - StackUtils.getNumber(params.get(1)).longValue(); + BigInteger result = StackUtils.getBigInteger(params.get(0)).subtract( + StackUtils.getBigInteger(params.get(1)) + ); inst.push(result); return ReadyFuture.DONE; } @@ -526,6 +530,28 @@ public class AsyncStackTester { } }); } + else if(op == StackOperation.TUPLE_SORT) { + return inst.popParam().flatMap(new Function>() { + @Override + public Future apply(Object param) { + final int listSize = StackUtils.getInt(param); + return inst.popParams(listSize).flatMap(new Function, Future>() { + @Override + public Future apply(List rawElements) { + List tuples = new ArrayList(listSize); + for(Object o : rawElements) { + tuples.add(Tuple.fromBytes((byte[])o)); + } + Collections.sort(tuples); + for(Tuple t : tuples) { + inst.push(t.pack()); + } + return ReadyFuture.DONE; + } + }); + } + }); + } else if(op == StackOperation.TUPLE_RANGE) { return inst.popParam().flatMap(new Function>() { @Override @@ -544,6 +570,48 @@ public class AsyncStackTester { } }); } + else if (op == StackOperation.ENCODE_FLOAT) { + return inst.popParam().map(new Function() { + @Override + public Void apply(Object param) { + byte[] fBytes = (byte[])param; + float value = ByteBuffer.wrap(fBytes).order(ByteOrder.BIG_ENDIAN).getFloat(); + inst.push(value); + return null; + } + }); + } + else if (op == StackOperation.ENCODE_DOUBLE) { + return inst.popParam().map(new Function() { + @Override + public Void apply(Object param) { + byte[] dBytes = (byte[])param; + double value = ByteBuffer.wrap(dBytes).order(ByteOrder.BIG_ENDIAN).getDouble(); + inst.push(value); + return null; + } + }); + } + else if (op == StackOperation.DECODE_FLOAT) { + return inst.popParam().map(new Function() { + @Override + public Void apply(Object param) { + float value = ((Number)param).floatValue(); + inst.push(ByteBuffer.allocate(4).order(ByteOrder.BIG_ENDIAN).putFloat(value).array()); + return null; + } + }); + } + else if (op == StackOperation.DECODE_DOUBLE) { + return inst.popParam().map(new Function() { + @Override + public Void apply(Object param) { + double value = ((Number)param).doubleValue(); + inst.push(ByteBuffer.allocate(8).order(ByteOrder.BIG_ENDIAN).putDouble(value).array()); + return null; + } + }); + } else if(op == StackOperation.UNIT_TESTS) { inst.context.db.options().setLocationCacheSize(100001); diff --git a/bindings/java/src/test/com/apple/cie/foundationdb/test/StackOperation.java b/bindings/java/src/test/com/apple/cie/foundationdb/test/StackOperation.java index 10c8c5c253..fc81d9104a 100644 --- a/bindings/java/src/test/com/apple/cie/foundationdb/test/StackOperation.java +++ b/bindings/java/src/test/com/apple/cie/foundationdb/test/StackOperation.java @@ -62,6 +62,11 @@ enum StackOperation { TUPLE_PACK, TUPLE_UNPACK, TUPLE_RANGE, + TUPLE_SORT, + ENCODE_FLOAT, + ENCODE_DOUBLE, + DECODE_FLOAT, + DECODE_DOUBLE, UNIT_TESTS, /* Possibly unimplemented */ LOG_STACK diff --git a/bindings/java/src/test/com/apple/cie/foundationdb/test/StackTester.java b/bindings/java/src/test/com/apple/cie/foundationdb/test/StackTester.java index 2d406006fb..5e1b79ba0a 100644 --- a/bindings/java/src/test/com/apple/cie/foundationdb/test/StackTester.java +++ b/bindings/java/src/test/com/apple/cie/foundationdb/test/StackTester.java @@ -20,6 +20,9 @@ package com.apple.cie.foundationdb.test; +import java.math.BigInteger; +import java.nio.ByteBuffer; +import java.nio.ByteOrder; import java.util.*; import com.apple.cie.foundationdb.Database; @@ -341,8 +344,9 @@ public class StackTester { } else if(op == StackOperation.SUB) { List params = inst.popParams(2).get(); - long result = StackUtils.getNumber(params.get(0)).longValue() - StackUtils.getNumber(params.get(1)).longValue(); - inst.push(result); + BigInteger a = StackUtils.getBigInteger(params.get(0)); + BigInteger b = StackUtils.getBigInteger(params.get(1)); + inst.push(a.subtract(b)); } else if(op == StackOperation.CONCAT) { List params = inst.popParams(2).get(); @@ -372,6 +376,18 @@ public class StackTester { inst.push(itemBytes); } } + else if (op == StackOperation.TUPLE_SORT) { + int listSize = StackUtils.getInt(inst.popParam().get()); + List rawElements = inst.popParams(listSize).get(); + List tuples = new ArrayList(listSize); + for(Object o : rawElements) { + tuples.add(Tuple.fromBytes((byte[])o)); + } + Collections.sort(tuples); + for(Tuple t : tuples) { + inst.push(t.pack()); + } + } else if(op == StackOperation.TUPLE_RANGE) { List params = inst.popParams(1).get(); int tupleSize = StackUtils.getInt(params, 0); @@ -381,6 +397,28 @@ public class StackTester { inst.push(range.begin); inst.push(range.end); } + else if (op == StackOperation.ENCODE_FLOAT) { + Object param = inst.popParam().get(); + byte[] fBytes = (byte[])param; + float value = ByteBuffer.wrap(fBytes).order(ByteOrder.BIG_ENDIAN).getFloat(); + inst.push(value); + } + else if (op == StackOperation.ENCODE_DOUBLE) { + Object param = inst.popParam().get(); + byte[] dBytes = (byte[])param; + double value = ByteBuffer.wrap(dBytes).order(ByteOrder.BIG_ENDIAN).getDouble(); + inst.push(value); + } + else if (op == StackOperation.DECODE_FLOAT) { + Object param = inst.popParam().get(); + float value = ((Number)param).floatValue(); + inst.push(ByteBuffer.allocate(4).order(ByteOrder.BIG_ENDIAN).putFloat(value).array()); + } + else if (op == StackOperation.DECODE_DOUBLE) { + Object param = inst.popParam().get(); + double value = ((Number)param).doubleValue(); + inst.push(ByteBuffer.allocate(8).order(ByteOrder.BIG_ENDIAN).putDouble(value).array()); + } else if(op == StackOperation.UNIT_TESTS) { try { inst.context.db.options().setLocationCacheSize(100001); @@ -566,7 +604,7 @@ public class StackTester { private static void logStack(final Database db, final Map entries, final byte[] prefix) { db.run(new Function() { - @Override + @Override public Void apply(Transaction tr) { for(Map.Entry it : entries.entrySet()) { byte[] pk = ByteArrayUtil.join(prefix, Tuple.from(it.getKey(), it.getValue().idx).pack()); diff --git a/bindings/java/src/test/com/apple/cie/foundationdb/test/StackUtils.java b/bindings/java/src/test/com/apple/cie/foundationdb/test/StackUtils.java index 6b1bfb11d0..ee681e03de 100644 --- a/bindings/java/src/test/com/apple/cie/foundationdb/test/StackUtils.java +++ b/bindings/java/src/test/com/apple/cie/foundationdb/test/StackUtils.java @@ -20,6 +20,7 @@ package com.apple.cie.foundationdb.test; +import java.math.BigInteger; import java.util.List; import com.apple.cie.foundationdb.FDBException; @@ -92,6 +93,14 @@ public class StackUtils { return ((Number)object); } + static BigInteger getBigInteger(Object object) { + if (object instanceof BigInteger) { + return (BigInteger)object; + } else { + return BigInteger.valueOf(((Number)object).longValue()); + } + } + static boolean getBoolean(Object o) { return getBoolean(o, null); } diff --git a/bindings/nodejs/lib/fdb.js b/bindings/nodejs/lib/fdb.js index ff8fa05826..86968007b3 100644 --- a/bindings/nodejs/lib/fdb.js +++ b/bindings/nodejs/lib/fdb.js @@ -41,8 +41,8 @@ module.exports = { apiVersion: function(version) { if(selectedApiVersion.value && version !== selectedApiVersion.value) throw new Error('Cannot select multiple different FDB API versions'); - if(version < 14) - throw new RangeError('FDB API versions before 14 are not supported'); + if(version < 500) + throw new RangeError('FDB API versions before 500 are not supported'); if(version > 500) throw new RangeError('Latest known FDB API version is 500'); diff --git a/bindings/nodejs/lib/tuple.js b/bindings/nodejs/lib/tuple.js index aaed94ac62..1fe1d1dcc5 100644 --- a/bindings/nodejs/lib/tuple.js +++ b/bindings/nodejs/lib/tuple.js @@ -24,6 +24,7 @@ var assert = require('assert'); var buffer = require('./bufferConversion'); var fdbUtil = require('./fdbUtil'); +var FDBError = require('./error'); var sizeLimits = new Array(8); @@ -43,6 +44,91 @@ var minInt = -Math.pow(2, 53); var nullByte = new Buffer('00', 'hex'); +var BYTES_CODE = 0x01; +var STRING_CODE = 0x02; +var NESTED_CODE = 0x05; +var INT_ZERO_CODE = 0x14; +var POS_INT_END = 0x1d; +var NEG_INT_END = 0x0b; +var FLOAT_CODE = 0x20; +var DOUBLE_CODE = 0x21; +var FALSE_CODE = 0x26; +var TRUE_CODE = 0x27; +var UUID_CODE = 0x30; + +function validateData(data, length) { + if(!(data instanceof Buffer) && !(data instanceof Array)) { + throw new TypeError('Data for FDB tuple type not array or buffer.'); + } else if(data.length != length) { + throw new RangeError('Data for FDB tuple type has length ' + data.length + ' instead of expected length ' + length); + } +} + +// If encoding and sign bit is 1 (negative), flip all of the bits. Otherwise, just flip sign. +// If decoding and sign bit is 0 (negative), flip all of the bits. Otherwise, just flip sign. +function adjustFloat(data, start, encode) { + if((encode && (data[start] & 0x80) === 0x80) || (!encode && (data[start] & 0x80) === 0x00)) { + for(var i = start; i < data.length; i++) { + data[i] = data[i] ^ 0xff; + } + } else { + data[start] = data[start] ^ 0x80; + } +} + +function Float(value) { + this.value = value; + this.toBytes = function () { + if (this.rawData !== undefined) { + return this.rawData; + } else { + var buf = new Buffer(4); + buf.writeFloatBE(this.value, 0); + return buf; + } + }; +} + +Float.fromBytes = function (buf) { + validateData(buf, 4); + var f = new Float(buf.readFloatBE(0)); + if(isNaN(f.value)) { + f.rawData = buf; + } + return f; +} + +function Double(value) { + this.value = value; + this.toBytes = function () { + if (this.rawData !== undefined) { + return this.rawData; + } else { + var buf = new Buffer(8); + buf.writeDoubleBE(this.value, 0); + return buf; + } + }; +} + +Double.fromBytes = function (buf) { + validateData(buf, 8); + var d = new Double(buf.readDoubleBE(0)); + if(isNaN(d.value)) { + d.rawData = buf; + } + return d; +} + +function UUID(data) { + if (data.length != 16) { + // There's a special code for this, so we check it first and throw the error if appropriate. + throw new FDBError("invalid_uuid_size", 2268); + } + validateData(data, 16); + this.data = new Buffer(data); +} + function findNullBytes(buf, pos, searchForTerminators) { var nullBytes = []; @@ -88,7 +174,7 @@ function encode(item, buf, pos) { var nullBytes = findNullBytes(item, 0); encodedString = new Buffer(2 + item.length + nullBytes.length); - encodedString[0] = unicode ? 2 : 1; + encodedString[0] = unicode ? STRING_CODE : BYTES_CODE; var srcPos = 0; var targetPos = 1; @@ -106,7 +192,7 @@ function encode(item, buf, pos) { } //64-bit integer - else if(item % 1 === 0) { + else if((typeof item === 'number' || item instanceof Number) && item % 1 === 0) { var negative = item < 0; var posItem = Math.abs(item); @@ -119,7 +205,7 @@ function encode(item, buf, pos) { if(item > maxInt || item < minInt) throw new RangeError('Cannot pack signed integer larger than 54 bits'); - var prefix = negative ? 20 - length : 20 + length; + var prefix = negative ? INT_ZERO_CODE - length : INT_ZERO_CODE + length; var outBuf = new Buffer(length+1); outBuf[0] = prefix; @@ -137,6 +223,74 @@ function encode(item, buf, pos) { return outBuf; } + // Floats + else if(item instanceof Float) { + var outBuf = new Buffer(5); + outBuf[0] = FLOAT_CODE; + if (isNaN(item.value) && item.rawData !== undefined) { + item.rawData.copy(outBuf, 1, 0, 4); + } else { + outBuf.writeFloatBE(item.value, 1); + } + adjustFloat(outBuf, 1, true); + return outBuf; + } + + // Doubles + else if(item instanceof Double) { + var outBuf = new Buffer(9); + outBuf[0] = DOUBLE_CODE; + if (isNaN(item.value) && item.rawData !== undefined) { + item.rawData.copy(outBuf, 1, 0, 8); + } else { + outBuf.writeDoubleBE(item.value, 1); + } + adjustFloat(outBuf, 1, true); + return outBuf; + } + + // UUIDs + else if(item instanceof UUID) { + var outBuf = new Buffer(17); + outBuf[0] = UUID_CODE; + item.data.copy(outBuf, 1); + return outBuf; + } + + // booleans + else if(item instanceof Boolean || typeof item === 'boolean') { + var outBuf = new Buffer(1); + var boolItem; + if(item instanceof Boolean) { + boolItem = item.valueOf(); + } else { + boolItem = item; + } + if(boolItem) { + outBuf[0] = TRUE_CODE; + } else { + outBuf[0] = FALSE_CODE; + } + return outBuf; + } + + // nested tuples + else if(item instanceof Array) { + var totalLength = 2; + var outArr = [new Buffer('05', 'hex')]; + for(var i = 0; i < item.length; ++i) { + if(item[i] === null) { + outArr.push(new Buffer('00ff', 'hex')); + totalLength += 2; + } else { + outArr.push(encode(item[i])); + totalLength += outArr[i+1].length; + } + } + outArr.push(new Buffer('00', 'hex')) + return Buffer.concat(outArr, totalLength); + } + else throw new TypeError('Packed element must either be a string, a buffer, an integer, or null'); } @@ -181,21 +335,27 @@ function decodeNumber(buf, offset, bytes) { return num; } -function decode(buf, pos) { +function decode(buf, pos, nested) { + if(typeof nested === 'undefined') nested = false; + var code = buf[pos]; var value; if(code === 0) { value = null; - pos++; + if(nested) { + pos += 2; + } else { + pos++; + } } - else if(code === 1 || code === 2) { + else if(code === BYTES_CODE || code === STRING_CODE) { var nullBytes = findNullBytes(buf, pos+1, true); var start = pos+1; var end = nullBytes[nullBytes.length-1]; - if(code === 2 && nullBytes.length === 1) { + if(code === STRING_CODE && nullBytes.length === 1) { value = buf.toString('utf8', start, end); } else { @@ -211,22 +371,60 @@ function decode(buf, pos) { } } - if(code === 2) + if(code === STRING_CODE) value = value.toString('utf8'); } pos = end + 1; } - else if(Math.abs(code-20) <= 7) { - if(code === 20) + else if(Math.abs(code-INT_ZERO_CODE) <= 7) { + if(code === INT_ZERO_CODE) value = 0; else - value = decodeNumber(buf, pos+1, code-20); + value = decodeNumber(buf, pos+1, code-INT_ZERO_CODE); - pos += Math.abs(20-code) + 1; + pos += Math.abs(INT_ZERO_CODE-code) + 1; } - else if(Math.abs(code-20) <= 8) + else if(Math.abs(code-INT_ZERO_CODE) <= 8) throw new RangeError('Cannot unpack signed integers larger than 54 bits'); + else if(code === FLOAT_CODE) { + var valBuf = new Buffer(4); + buf.copy(valBuf, 0, pos+1, pos+5); + adjustFloat(valBuf, 0, false); + value = Float.fromBytes(valBuf); + pos += 5; + } + else if(code === DOUBLE_CODE) { + var valBuf = new Buffer(8); + buf.copy(valBuf, 0, pos+1, pos+9); + adjustFloat(valBuf, 0, false); + value = Double.fromBytes(valBuf); + pos += 9; + } + else if(code === UUID_CODE) { + var valBuf = new Buffer(16); + buf.copy(valBuf, 0, pos+1, pos+17); + value = new UUID(valBuf); + pos += 17; + } + else if(code === FALSE_CODE) { + pos++; + value = false; + } + else if(code === TRUE_CODE) { + pos++; + value = true; + } + else if(code === NESTED_CODE) { + pos++; + value = [] + while (buf[pos] != 0 || pos+1 < buf.length && buf[pos+1] === 0xff) { + var nestedVal = decode(buf, pos, true) + pos = nestedVal.pos + value.push(nestedVal.value) + } + pos++; + } else throw new TypeError('Unknown data type in DB: ' + buf + ' at ' + pos); @@ -252,4 +450,31 @@ function range(arr) { return { begin: Buffer.concat([packed, nullByte]), end: Buffer.concat([packed, new Buffer('ff', 'hex')]) }; } -module.exports = {pack: pack, unpack: unpack, range: range}; +function compare(arr1, arr2) { + // NOTE: There is built-in comparison function included in 0.11.13 that we might want to switch to. + var buf1 = pack(arr1); + var buf2 = pack(arr2); + var pos = 0; + + while(pos < buf1.length && pos < buf2.length) { + if(buf1[pos] != buf2[pos]) { + if(buf1[pos] < buf2[pos]) { + return -1; + } else { + return 1; + } + } + pos += 1; + } + + // The two arrays begin with a common prefix. + if(buf1.length < buf2.length) { + return -1; + } else if(buf1.length == buf2.length) { + return 0; + } else { + return 1; + } +} + +module.exports = {pack: pack, unpack: unpack, range: range, compare: compare, Float: Float, Double: Double, UUID: UUID}; diff --git a/bindings/nodejs/tests/tester.js b/bindings/nodejs/tests/tester.js index a0e93e934e..850989ff0f 100755 --- a/bindings/nodejs/tests/tester.js +++ b/bindings/nodejs/tests/tester.js @@ -620,12 +620,50 @@ function processOperation(context, inst, cb) { }); })(promiseCb); } + else if(inst.op === 'TUPLE_SORT') { + inst.pop() + .then(function(numParams) { + return inst.pop({count: numParams}) + .then(function(params) { + var tuples = []; + for(var i = 0; i < params.length; ++i) + tuples.push(fdb.tuple.unpack(params[i])); + tuples.sort(fdb.tuple.compare); + for(var i = 0; i < tuples.length; ++i) + inst.push(fdb.tuple.pack(tuples[i])); + }); + })(promiseCb); + } else if(inst.op === 'SUB') { inst.pop({count: 2}) .then(function(params) { inst.push(params[0] - params[1]); })(promiseCb); } + else if(inst.op === 'ENCODE_FLOAT') { + inst.pop() + .then(function(fBytes) { + inst.push(fdb.tuple.Float.fromBytes(fBytes)); + })(promiseCb); + } + else if(inst.op === 'ENCODE_DOUBLE') { + inst.pop() + .then(function(dBytes) { + inst.push(fdb.tuple.Double.fromBytes(dBytes)); + })(promiseCb); + } + else if(inst.op === 'DECODE_FLOAT') { + inst.pop() + .then(function(fVal) { + inst.push(fVal.toBytes()); + })(promiseCb); + } + else if(inst.op === 'DECODE_DOUBLE') { + inst.pop() + .then(function(dVal) { + inst.push(dVal.toBytes()); + })(promiseCb); + } else if(inst.op === 'CONCAT') { inst.pop({count: 2}) .then(function(params) { diff --git a/bindings/nodejs/tests/tuple_test.js b/bindings/nodejs/tests/tuple_test.js index 7edd08b3c9..e430c964c4 100755 --- a/bindings/nodejs/tests/tuple_test.js +++ b/bindings/nodejs/tests/tuple_test.js @@ -35,6 +35,44 @@ catch(err) { console.log(fdb.tuple.pack([0xff * 0xff])); console.log(fdb.tuple.pack([0xffffffff + 100 ])); +console.log(fdb.buffer.printable(fdb.tuple.pack(['begin', [true, null, false], 'end']))) console.log(fdb.tuple.unpack(fdb.buffer.fromByteLiteral('\x1a\xff\xff\xff\xff\xff\xff'))); -console.log(fdb.tuple.unpack(fdb.tuple.pack(['TEST', 'herp', 1, -10, 393493, '\u0000abc', 0xffffffff + 100]))); +console.log(fdb.tuple.unpack(fdb.tuple.pack(['TEST', 'herp', 1, -10, 393493, '\u0000abc', 0xffffffff + 100, true, false, [new Boolean(true), null, new Boolean(false), 0, 'asdf'], null]))); +console.log(fdb.buffer.printable(fdb.tuple.pack([[[[['three']]], 'two'], 'one']))) console.log(fdb.tuple.range(['TEST', 1])); +console.log(fdb.buffer.printable(fdb.tuple.pack([fdb.tuple.Float.fromBytes(new Buffer('402df854', 'hex')), fdb.tuple.Double.fromBytes(new Buffer('4005BF0A8B145769', 'hex')), new fdb.tuple.UUID(new Buffer('deadc0deba5eba115ca1ab1edeadc0de', 'hex'))]))) +console.log(fdb.tuple.unpack(fdb.tuple.pack([fdb.tuple.Float.fromBytes(new Buffer('2734236f', 'hex'))]))) + +tuples = [ + [1,2], + [1], + [2], + [true], + [false], + [1,true], + [1,false], + [1, []], + [1, [null]], + [1, [0]], + [1, [1]], + [1, [0,1,2]], + [null], + [] +]; +tuples.sort(fdb.tuple.compare); +console.log(tuples); + +tuples = [ + [fdb.tuple.Float.fromBytes(new Buffer('2734236f', 'hex'))], // A really small value. + [fdb.tuple.Float.fromBytes(new Buffer('80000000', 'hex'))], // -0.0 + [new fdb.tuple.Float(0.0)], + [new fdb.tuple.Float(3.14)], + [new fdb.tuple.Float(-3.14)], + [new fdb.tuple.Float(2.7182818)], + [new fdb.tuple.Float(-2.7182818)], + [fdb.tuple.Float.fromBytes(new Buffer('7f800000', 'hex'))], // Infinity + [fdb.tuple.Float.fromBytes(new Buffer('7fffffff', 'hex'))], // NaN + [fdb.tuple.Float.fromBytes(new Buffer('ffffffff', 'hex'))], // -NaN +]; +tuples.sort(fdb.tuple.compare); +console.log(tuples); diff --git a/bindings/python/fdb/tuple.py b/bindings/python/fdb/tuple.py index c64873956c..780f7295dc 100644 --- a/bindings/python/fdb/tuple.py +++ b/bindings/python/fdb/tuple.py @@ -20,13 +20,30 @@ # FoundationDB Python API -import struct, math +import ctypes, uuid, struct, math from bisect import bisect_left from fdb import six +import fdb _size_limits = tuple( (1 << (i*8))-1 for i in range(9) ) +# Define type codes: +NULL_CODE = 0x00 +BYTES_CODE = 0x01 +STRING_CODE = 0x02 +NESTED_CODE = 0x05 +INT_ZERO_CODE = 0x14 +POS_INT_END = 0x1d +NEG_INT_START = 0x0b +FLOAT_CODE = 0x20 +DOUBLE_CODE = 0x21 +FALSE_CODE = 0x26 +TRUE_CODE = 0x27 +UUID_CODE = 0x30 + +# Reserved: Codes 0x03, 0x04, 0x23, and 0x24 are reserved for historical reasons. + def _find_terminator( v, pos ): # Finds the start of the next terminator [\x00]![\xff] or the end of v while True: @@ -37,76 +54,183 @@ def _find_terminator( v, pos ): return pos pos += 2 +# If encoding and sign bit is 1 (negative), flip all of the bits. Otherwise, just flip sign. +# If decoding and sign bit is 0 (negative), flip all of the bits. Otherwise, just flip sign. +def _float_adjust( v, encode ): + if encode and six.indexbytes(v, 0) & 0x80 != 0x00: + return b''.join(map(lambda x: six.int2byte(x ^ 0xff), six.iterbytes(v))) + elif not encode and six.indexbytes(v, 0) & 0x80 != 0x80: + return b''.join(map(lambda x: six.int2byte(x ^ 0xff), six.iterbytes(v))) + else: + return six.int2byte(six.indexbytes(v, 0) ^ 0x80) + v[1:] + +class SingleFloat(object): + def __init__(self, value): + if isinstance(value, float): + # Restrict to the first 4 bytes (essentially) + self.value = ctypes.c_float(value).value + elif isinstance(value, ctypes.c_float): + self.value = value.value + elif isinstance(value, six.integertypes): + self.value = ctypes.c_float(value).value + else: + raise ValueError("Incompatible type for single-precision float: " + repr(value)) + + # Comparisons + def __eq__(self, other): + if isinstance(other, SingleFloat): + return _compare_floats(self.value, other.value) == 0 + else: + return False + + def __ne__(self, other): + return not (self == other) + + def __lt__(self, other): + return _compare_floats(self.value, other.value) < 0 + + def __le__(self, other): + return _compare_floats(self.value, other.value) <= 0 + + def __gt__(self, other): + return not (self <= other) + + def __ge__(self, other): + return not (self < other) + + def __str__(self): + return str(self.value) + + def __repr__(self): + return "SingleFloat(" + str(self.value) + ")" + + def __hash__(self): + # Left-circulate the child hash to make hash(self) != hash(self.value) + v_hash = hash(self.value) + if v_hash >= 0: + return (v_hash >> 16) + ((v_hash & 0xFFFF) << 16) + else: + return ((v_hash >> 16) + 1) - ((abs(v_hash) & 0xFFFF) << 16) + + def __nonzero__(self): + return bool(self.value) + def _decode(v, pos): code = six.indexbytes(v, pos) - if code == 0: + if code == NULL_CODE: return None, pos+1 - elif code == 1: + elif code == BYTES_CODE: end = _find_terminator(v, pos+1) return v[pos+1:end].replace(b"\x00\xFF", b"\x00"), end+1 - elif code == 2: + elif code == STRING_CODE: end = _find_terminator(v, pos+1) return v[pos+1:end].replace(b"\x00\xFF", b"\x00").decode("utf-8"), end+1 - elif code >= 20 and code <= 28: + elif code >= INT_ZERO_CODE and code < POS_INT_END: n = code - 20 end = pos + 1 + n return struct.unpack(">Q", b'\x00'*(8-n) + v[pos+1:end])[0], end - elif code >= 12 and code < 20: + elif code > NEG_INT_START and code < INT_ZERO_CODE: n = 20 - code end = pos + 1 + n return struct.unpack(">Q", b'\x00'*(8-n) + v[pos+1:end])[0]-_size_limits[n], end - elif code == 29: # 0x1d; Positive 9-255 byte integer + elif code == POS_INT_END: # 0x1d; Positive 9-255 byte integer length = six.indexbytes(v, pos+1) val = 0 for i in _range(length): val = val << 8 val += six.indexbytes(v, pos+2+i) return val, pos+2+length - elif code == 11: # 0x0b; Negative 9-255 byte integer + elif code == NEG_INT_START: # 0x0b; Negative 9-255 byte integer length = six.indexbytes(v, pos+1)^0xff val = 0 for i in _range(length): val = val << 8 val += six.indexbytes(v, pos+2+i) return val - (1<<(length*8)) + 1, pos+2+length + elif code == FLOAT_CODE: + return SingleFloat(struct.unpack(">f", _float_adjust(v[pos+1:pos+5], False))[0]), pos+5 + elif code == DOUBLE_CODE: + return struct.unpack(">d", _float_adjust(v[pos+1:pos+9], False))[0], pos+9 + elif code == UUID_CODE: + return uuid.UUID(bytes=v[pos+1:pos+17]), pos+17 + elif code == FALSE_CODE: + if hasattr(fdb, "_version") and fdb._version < 500: + raise ValueError("Invalid API version " + str(fdb._version) + " for boolean types") + return False, pos+1 + elif code == TRUE_CODE: + if hasattr(fdb, "_version") and fdb._version < 500: + raise ValueError("Invalid API version " + str(fdb._version) + " for boolean types") + return True, pos+1 + elif code == NESTED_CODE: + ret = [] + end_pos = pos+1 + while end_pos < len(v): + if six.indexbytes(v, end_pos) == 0x00: + if end_pos+1 < len(v) and six.indexbytes(v, end_pos+1) == 0xff: + ret.append(None) + end_pos += 2 + else: + break + else: + val, end_pos = _decode(v, end_pos) + ret.append(val) + return tuple(ret), end_pos+1 else: raise ValueError("Unknown data type in DB: " + repr(v)) -def _encode(value): +def _encode(value, nested=False): # returns [code][data] (code != 0xFF) # encoded values are self-terminating # sorting need to work too! if value == None: # ==, not is, because some fdb.impl.Value are equal to None - return b'\x00' + if nested: + return b''.join([six.int2byte(NULL_CODE), six.int2byte(0xff)]) + else: + return b''.join([six.int2byte(NULL_CODE)]) elif isinstance(value, bytes): # also gets non-None fdb.impl.Value - return b'\x01' + value.replace(b'\x00', b'\x00\xFF') + b'\x00' + return six.int2byte(BYTES_CODE) + value.replace(b'\x00', b'\x00\xFF') + b'\x00' elif isinstance(value, six.text_type): - return b'\x02' + value.encode('utf-8').replace(b'\x00', b'\x00\xFF') + b'\x00' - elif isinstance(value, six.integer_types): + return six.int2byte(STRING_CODE) + value.encode('utf-8').replace(b'\x00', b'\x00\xFF') + b'\x00' + elif isinstance(value, six.integer_types) and (not isinstance(value, bool) or (hasattr(fdb, '_version') and fdb._version < 500)): if value == 0: - return b'\x14' + return b''.join([six.int2byte(INT_ZERO_CODE)]) elif value > 0: if value >= _size_limits[-1]: length = (value.bit_length()+7)//8 - data = [b'\x1d', six.int2byte(length)] + data = [six.int2byte(POS_INT_END), six.int2byte(length)] for i in _range(length-1,-1,-1): data.append(six.int2byte( (value>>(8*i))&0xff )) return b''.join(data) n = bisect_left( _size_limits, value ) - return six.int2byte(20 + n) + struct.pack( ">Q", value )[-n:] + return six.int2byte(INT_ZERO_CODE + n) + struct.pack( ">Q", value )[-n:] else: if -value >= _size_limits[-1]: length = (value.bit_length()+7)//8 value += (1<<(length*8)) - 1 - data = [b'\x0b', six.int2byte(length^0xff)] + data = [six.int2byte(NEG_INT_START), six.int2byte(length^0xff)] for i in _range(length-1,-1,-1): data.append(six.int2byte( (value>>(8*i))&0xff )) return b''.join(data) n = bisect_left( _size_limits, -value ) maxv = _size_limits[n] - return six.int2byte(20 - n) + struct.pack( ">Q", maxv+value)[-n:] + return six.int2byte(INT_ZERO_CODE - n) + struct.pack( ">Q", maxv+value)[-n:] + elif isinstance(value, ctypes.c_float) or isinstance(value, SingleFloat): + return six.int2byte(FLOAT_CODE) + _float_adjust(struct.pack(">f", value.value), True) + elif isinstance(value, ctypes.c_double): + return six.int2byte(DOUBLE_CODE) + _float_adjust(struct.pack(">d", value.value), True) + elif isinstance(value, float): + return six.int2byte(DOUBLE_CODE) + _float_adjust(struct.pack(">d", value), True) + elif isinstance(value, uuid.UUID): + return six.int2byte(UUID_CODE) + value.bytes + elif isinstance(value, bool): + if value: + return b''.join([six.int2byte(TRUE_CODE)]) + else: + return b''.join([six.int2byte(FALSE_CODE)]) + elif isinstance(value, tuple) or isinstance(value, list): + return b''.join([six.int2byte(NESTED_CODE)] + list(map(lambda x: _encode(x, True), value)) + [six.int2byte(0x00)]) else: raise ValueError("Unsupported data type: " + str(type(value))) @@ -140,3 +264,91 @@ def range(t): return slice( p+b'\x00', p+b'\xff') + +def _code_for(value): + if value == None: + return NULL_CODE + elif isinstance(value, bytes): + return BYTES_CODE + elif isinstance(value, six.text_type): + return STRING_CODE + elif (not hasattr(fdb, '_version') or fdb._version >= 500) and isinstance(value, bool): + return FALSE_CODE + elif isinstance(value, six.integer_types): + return INT_ZERO_CODE + elif isinstance(value, ctypes.c_float) or isinstance(value, SingleFloat): + return FLOAT_CODE + elif isinstance(value, ctypes.c_double) or isinstance(value, float): + return DOUBLE_CODE + elif isinstance(value, uuid.UUID): + return UUID_CODE + elif isinstance(value, tuple) or isinstance(value, list): + return NESTED_CODE + else: + raise ValueError("Unsupported data type: " + str(type(value))) + +def _compare_floats(f1, f2): + sign1 = int(math.copysign(1, f1)) + sign2 = int(math.copysign(1, f2)) + + # This business with signs is to deal with negative zero, NaN, and infinity. + if sign1 < sign2: + # f1 is negative and f2 is positive. + return -1 + elif sign1 > sign2: + # f1 is positive and f2 is negative. + return 1 + + if not math.isnan(f1) and not math.isnan(f2): + return -1 if f1 < f2 else 0 if f1 == f2 else 1 + + # There are enough edge cases that bit comparison is safer. + bytes1 = struct.pack(">d", f1) + bytes2 = struct.pack(">d", f2) + return sign1*(-1 if bytes1 < bytes2 else 0 if bytes1 == bytes2 else 1) + +def _compare_values(value1, value2): + code1 = _code_for(value1) + code2 = _code_for(value2) + + if code1 < code2: + return -1 + elif code1 > code2: + return 1 + + # Compatible types. + if code1 == NULL_CODE: + return 0 + elif code1 == STRING_CODE: + encoded1 = value1.encode('utf-8') + encoded2 = value2.encode('utf-8') + return -1 if encoded1 < encoded2 else 0 if encoded1 == encoded2 else 1 + elif code1 == FLOAT_CODE: + f1 = value1 if isinstance(value1, SingleFloat) else SingleFloat(value1.value) + f2 = value2 if isinstance(value2, SingleFloat) else SingleFloat(value2.value) + return -1 if f1 < f2 else 0 if f1 == f2 else 1 + elif code1 == DOUBLE_CODE: + d1 = value1.value if isinstance(value1, ctypes.c_double) else value1 + d2 = value2.value if isinstance(value2, ctypes.c_double) else value2 + return _compare_floats(d1, d2) + elif code1 == NESTED_CODE: + return compare(value1, value2) + else: + # Booleans, UUIDs, and integers can just use standard comparison. + return -1 if value1 < value2 else 0 if value1 == value2 else 1 + +# compare element by element and return -1 if t1 < t2 or 1 if t1 > t2 or 0 if t1 == t2 +def compare(t1, t2): + i = 0 + while i < len(t1) and i < len(t2): + c = _compare_values(t1[i], t2[i]) + if c != 0: + return c + i += 1 + + if i < len(t1): + return 1 + elif i < len(t2): + return -1 + else: + return 0 diff --git a/bindings/python/tests/tester.py b/bindings/python/tests/tester.py index b825872d28..54285b1f48 100755 --- a/bindings/python/tests/tester.py +++ b/bindings/python/tests/tester.py @@ -20,8 +20,10 @@ # +import ctypes import sys import os +import struct import threading import time import random @@ -454,12 +456,38 @@ class Tester: elif inst.op == six.u("TUPLE_UNPACK"): for i in fdb.tuple.unpack( inst.pop() ): inst.push(fdb.tuple.pack((i,))) + elif inst.op == six.u("TUPLE_SORT"): + count = inst.pop() + items = inst.pop(count) + unpacked = map(fdb.tuple.unpack, items) + if six.PY3: + sorted_items = sorted(unpacked, key=fdb.tuple.pack) + else: + sorted_items = sorted(unpacked, cmp=fdb.tuple.compare) + for item in sorted_items: + inst.push(fdb.tuple.pack(item)) elif inst.op == six.u("TUPLE_RANGE"): count = inst.pop() items = inst.pop(count) r = fdb.tuple.range( tuple(items) ) inst.push(r.start) inst.push(r.stop) + elif inst.op == six.u("ENCODE_FLOAT"): + f_bytes = inst.pop() + f = struct.unpack(">f", f_bytes)[0] + inst.push(fdb.tuple.SingleFloat(f)) + elif inst.op == six.u("ENCODE_DOUBLE"): + d_bytes = inst.pop() + d = struct.unpack(">d", d_bytes)[0] + inst.push(d) + elif inst.op == six.u("DECODE_FLOAT"): + f = inst.pop() + f_bytes = struct.pack(">f", f.value) + inst.push(f_bytes) + elif inst.op == six.u("DECODE_DOUBLE"): + d = inst.pop() + d_bytes = struct.pack(">d", d) + inst.push(d_bytes) elif inst.op == six.u("START_THREAD"): t = Tester( self.db, inst.pop() ) thr = threading.Thread(target=t.run) diff --git a/bindings/python/tests/tuple_tests.py b/bindings/python/tests/tuple_tests.py index 4550da87a4..c0e46aa3cb 100644 --- a/bindings/python/tests/tuple_tests.py +++ b/bindings/python/tests/tuple_tests.py @@ -20,54 +20,107 @@ # -import sys, random +import ctypes, sys, random, struct, unicodedata, math, uuid _range = range -from fdb.tuple import pack, unpack, range +from fdb.tuple import pack, unpack, range, compare, SingleFloat from fdb import six from fdb.six import u +def randomUnicode(): + while True: + c = random.randint(0, 0xffff) + if unicodedata.category(unichr(c))[0] in 'LMNPSZ': + return unichr(c) + def randomElement(): - r = random.randint(0,4) + r = random.randint(0,9) if r == 0: - chars = [b'\x00', b'\x01', b'a', b'7', b'\xfe', b'\ff'] - return b''.join([random.choice(chars) for c in _range(random.randint(0, 5))]) + if random.random() < 0.5: + chars = [b'\x00', b'\x01', b'a', b'7', b'\xfe', b'\ff'] + return b''.join([random.choice(chars) for c in _range(random.randint(0, 5))]) + else: + return b''.join([six.int2byte(random.randint(0,255)) for _ in _range(random.randint(0,10))]) elif r == 1: - chars = [u('\x00'), u('\x01'), u('a'), u('7'), u('\xfe'), u('\ff'), u('\u0000'), u('\u0001'), u('\uffff'), u('\uff00')] - return u('').join([random.choice(chars) for c in _range(random.randint(0, 10))]) + if random.random() < 0.5: + chars = [u('\x00'), u('\x01'), u('a'), u('7'), u('\xfe'), u('\ff'), u('\u0000'), u('\u0001'), u('\uffff'), u('\uff00'), u('\U0001f4a9')] + return u('').join([random.choice(chars) for c in _range(random.randint(0, 10))]) + else: + return u('').join([randomUnicode() for _ in _range(random.randint(0, 10))]) elif r == 2: return random.choice([-1, 1]) * min(2**random.randint(0, 2040) + random.randint(-10, 10), 2**2040 - 1) elif r == 3: return random.choice([-1, 1]) * 2**random.randint(0, 64) + random.randint(-10, 10) elif r == 4: return None + elif r == 5: + ret = random.choice([float('-nan'), float('-inf'), -0.0, 0.0, float('inf'), float('nan')]) + if random.random() < 0.5: + return SingleFloat(ret) + else: + return ret + elif r == 6: + is_double = random.random() < 0.5 + byte_str = b''.join([six.int2byte(random.randint(0,255)) for _ in _range(8 if is_double else 4)]) + if is_double: + return struct.unpack(">d", byte_str)[0] + else: + return SingleFloat(struct.unpack(">f", byte_str)[0]) + elif r == 7: + return random.random() < 0.5 + elif r == 8: + return uuid.uuid4() + elif r == 9: + return [randomElement() for _ in _range(random.randint(0,5))] def randomTuple(): return tuple( randomElement() for x in _range(random.randint(0,4)) ) def isprefix(a,b): - return tupleorder(a) == tupleorder(b[:len(a)]) + return compare(a, b[:len(a)]) == 0 -def torder(x): - if x == None: - return 0 - elif type(x) == type(b''): - return 1 - elif isinstance(x, six.text_type): - return 2 - elif isinstance(x, six.integer_types): - return 3 - raise Exception("Unknown type") -def tupleorder(t): - return tuple( (torder(e),e) for e in t ) +def find_bad_sort(a, b): + for x1 in a: + for x2 in b: + if compare(x1, x2) < 0 and pack(x1) >= pack(x2): + return (x1, x2) + return None + +def equalEnough(t1, t2): + if len(t1) != len(t2): return False + + for i in _range(len(t1)): + e1 = t1[i] + e2 = t2[i] + + if isinstance(e1, SingleFloat): + if not isinstance(e2, SingleFloat): return False + return ctypes.c_float(e1.value).value == ctypes.c_float(e2.value).value + elif isinstance(e1, list) or isinstance(e2, tuple): + if not (isinstance(e2, list) or isinstance(e2, tuple)): return False + if not equalEnough(e1, e2): return False + else: + if e1 != e2: return False + + return True def tupleTest(N=10000): someTuples = [ randomTuple() for i in _range(N) ] - a = sorted(someTuples, key=tupleorder) + a = sorted(someTuples, cmp=compare) b = sorted(someTuples, key=pack) - assert a == b + + if not a == b: + problem = find_bad_sort(a, b) + if problem: + print("Bad sort:\n %s\n %s" % (problem[0], problem[1])) + print("Bytes:\n %s\n %s" % (repr(pack(problem[0])), repr(pack(problem[1])))) + #print("Tuple order:\n %s\n %s" % (tupleorder(problem[0]), tupleorder(problem[1]))) + return False + else: + print("Sorts unequal but every pair correct") + return False print("Sort %d OK" % N) @@ -75,26 +128,36 @@ def tupleTest(N=10000): t = randomTuple() t2 = t + (randomElement(),) t3 = randomTuple() - try: - assert(unpack(pack(t)) == t) - r = range(t) - assert not (r.start <= pack(t) < r.stop) - assert (r.start <= pack(t2) < r.stop) + if not compare(unpack(pack(t)), t) == 0: + print("unpack . pack /= identity:\n Orig: %s\n Bytes: %s\n New: %s" % (t, repr(pack(t)), unpack(pack(t)))) + return False - if not isprefix(t, t3): - assert not (r.start <= pack(t3) <= r.stop) + r = range(t) + if r.start <= pack(t) < r.stop: + print("element within own range:\n Tuple: %s\n Bytes: %s\n Start: %s\n Stop: %s" % (t, repr(pack(t)), repr(r.start), repr(r.stop))) + if not r.start <= pack(t2) < r.stop: + print("prefixed element not in range:\n Tuple: %s\n Bytes: %s\n Prefixed: %s\n Bytes: %s" % (t, repr(pack(t)), t2, repr(pack(t2)))) + return False - assert (tupleorder(t) < tupleorder(t3)) == (pack(t) < pack(t3)) - except: - print (repr(t), repr(t2), repr(t3)) - raise + if not isprefix(t, t3): + if r.start <= pack(t3) <= r.stop: + print("non-prefixed element in range:\n Tuple: %s\n Bytes: %s\n Other: %s\n Bytes: %s" % (t, repr(pack(t)), t3, repr(pack(t3)))) + return False + + if (compare(t, t3) < 0) != (pack(t) < pack(t3)): + print("Bad comparison:\n Tuple: %s\n Bytes: %s\n Other: %s\n Bytes: %s" % (t, repr(pack(t)), t3, repr(pack(t3)))) + return False + if not pack(t) < pack(t2): + print("Prefix not before prefixed:\n Tuple: %s\n Bytes: %s\n Other: %s\n Bytes: %s" % (t, repr(pack(t)), t2, repr(pack(t2)))) + return False print ("Tuple check %d OK" % N) + return True # test: # a = ('\x00a', -2, 'b\x01', 12345, '') # assert(a==fdbtuple.unpack(fdbtuple.pack(a))) if __name__=='__main__': - tupleTest() + assert tupleTest(10000) diff --git a/bindings/ruby/lib/fdbtuple.rb b/bindings/ruby/lib/fdbtuple.rb index 2349f367b3..c392a9cf0f 100644 --- a/bindings/ruby/lib/fdbtuple.rb +++ b/bindings/ruby/lib/fdbtuple.rb @@ -29,6 +29,60 @@ module FDB module Tuple @@size_limits = (0..8).map {|x| (1 << (x*8)) - 1} + # Type codes + @@NULL_CODE = 0x00 + @@BYTES_CODE = 0x01 + @@STRING_CODE = 0x02 + @@NESTED_CODE = 0x05 + @@INT_ZERO_CODE = 0x14 + @@POS_INT_END = 0x1c + @@NEG_INT_START = 0x0c + @@FLOAT_CODE = 0x20 + @@DOUBLE_CODE = 0x21 + @@FALSE_CODE = 0x26 + @@TRUE_CODE = 0x27 + @@UUID_CODE = 0x30 + + class UUID + def initialize(data) + if data.length != 16 + raise Error.new(2268) # invalid_uuid_size + end + @data=data.slice(0,16) + end + def data + @data + end + def <=> (other) + self.data <=> other.data + end + def to_s + self.data.each_byte.map { |b| b.to_s(16) } .join + end + end + + class SingleFloat + def initialize(value) + if value.kind_of? Float + @value=value + elsif value.kind_of? Integer + @value=value.to_f + else + raise ArgumentError, "Invalid value type for SingleFloat: " + value.class.name + end + @value=value + end + def value + @value + end + def <=> (other) + Tuple._compare_floats(self.value, other.value, false) + end + def to_s + self.value.to_s + end + end + def self.find_terminator(v, pos) while true pos = v.index("\x00", pos) @@ -42,24 +96,62 @@ module FDB end private_class_method :find_terminator + def self.float_adjust(v, pos, length, encode) + if (encode and v[pos].ord & 0x80 != 0x00) or (not encode and v[pos].ord & 0x80 == 0x00) + v.slice(pos, length).chars.map { |b| (b.ord ^ 0xff).chr } .join + else + ret = v.slice(pos, length) + ret[0] = (ret[0].ord ^ 0x80).chr + ret + end + end + private_class_method :float_adjust + def self.decode(v, pos) code = v.getbyte(pos) - if code == 0 + if code == @@NULL_CODE [nil, pos+1] - elsif code == 1 + elsif code == @@BYTES_CODE epos = find_terminator(v, pos+1) [v.slice(pos+1, epos-pos-1).gsub("\x00\xFF", "\x00"), epos+1] - elsif code == 2 + elsif code == @@STRING_CODE epos = find_terminator(v, pos+1) [v.slice(pos+1, epos-pos-1).gsub("\x00\xFF", "\x00").force_encoding("UTF-8"), epos+1] - elsif code >= 20 && code <= 28 - n = code - 20 + elsif code >= @@INT_ZERO_CODE && code <= @@POS_INT_END + n = code - @@INT_ZERO_CODE [("\x00" * (8-n) + v.slice(pos+1, n)).unpack("Q>")[0], pos+n+1] - elsif code >= 12 and code < 20 - n = 20 - code + elsif code >= @@NEG_INT_START and code < @@INT_ZERO_CODE + n = @@INT_ZERO_CODE - code [("\x00" * (8-n) + v.slice(pos+1, n)).unpack("Q>")[0]-@@size_limits[n], pos+n+1] + elsif code == @@FALSE_CODE + [false, pos+1] + elsif code == @@TRUE_CODE + [true, pos+1] + elsif code == @@FLOAT_CODE + [SingleFloat.new(float_adjust(v, pos+1, 4, false).unpack("g")[0]), pos+5] + elsif code == @@DOUBLE_CODE + [float_adjust(v, pos+1, 8, false).unpack("G")[0], pos+9] + elsif code == @@UUID_CODE + [UUID.new(v.slice(pos+1, 16)), pos+17] + elsif code == @@NESTED_CODE + epos = pos+1 + nested = [] + while epos < v.length + if v.getbyte(epos) == @@NULL_CODE + if epos+1 < v.length and v.getbyte(epos+1) == 0xFF + nested << nil + epos += 2 + else + break + end + else + r, epos = decode(v, epos) + nested << r + end + end + [nested, epos+1] else - raise "Unknown data type in DB: " + v + raise "Unknown data type in DB: " + code.ord.to_s end end private_class_method :decode @@ -74,21 +166,25 @@ module FDB end private_class_method :bisect_left - def self.encode(v) - if !v - "\x00" + def self.encode(v, nested=false) + if v.nil? + if nested + "\x00\xFF" + else + @@NULL_CODE.chr + end elsif v.kind_of? String if v.encoding == Encoding::BINARY || v.encoding == Encoding::ASCII - 1.chr + v.gsub("\x00", "\x00\xFF") + 0.chr + @@BYTES_CODE.chr + v.gsub("\x00", "\x00\xFF") + 0.chr elsif v.encoding == Encoding::UTF_8 - 2.chr + v.dup.force_encoding("BINARY").gsub("\x00", "\x00\xFF") + 0.chr + @@STRING_CODE.chr + v.dup.force_encoding("BINARY").gsub("\x00", "\x00\xFF") + 0.chr else raise ArgumentError, "unsupported encoding #{v.encoding.name}" end elsif v.kind_of? Integer raise RangeError, "value outside inclusive range -2**64+1 to 2**64-1" if v < -2**64+1 || v > 2**64-1 if v == 0 - 20.chr + @@INT_ZERO_CODE.chr elsif v > 0 n = bisect_left( @@size_limits, v ) (20+n).chr + [v].pack("Q>").slice(8-n, n) @@ -96,6 +192,18 @@ module FDB n = bisect_left( @@size_limits, -v ) (20-n).chr + [@@size_limits[n]+v].pack("Q>").slice(8-n, n) end + elsif v.kind_of? TrueClass + @@TRUE_CODE.chr + elsif v.kind_of? FalseClass + @@FALSE_CODE.chr + elsif v.kind_of? SingleFloat + @@FLOAT_CODE.chr + float_adjust([v.value].pack("g"), 0, 4, true) + elsif v.kind_of? Float + @@DOUBLE_CODE.chr + float_adjust([v].pack("G"), 0, 8, true) + elsif v.kind_of? UUID + @@UUID_CODE.chr + v.data + elsif v.kind_of? Array + @@NESTED_CODE.chr + (v.map { |el| encode(el, true).force_encoding("BINARY") }).join + 0.chr else raise ArgumentError, "unsupported type #{v.class}" end @@ -127,5 +235,73 @@ module FDB p = pack(tuple) [p+"\x00", p+"\xFF"] end + + def self._code_for(v) + if v.nil? + @@NULL_CODE + elsif v.kind_of? String + if v.encoding == Encoding::BINARY || v.encoding == Encoding::ASCII + @@BYTES_CODE + elsif v.encoding == Encoding::UTF_8 + @@STRING_CODE + else + raise ArgumentError, "unsupported encoding #{v.encoding.name}" + end + elsif v.kind_of? Integer + @@INT_ZERO_CODE + elsif v.kind_of? TrueClass + @@TRUE_CODE + elsif v.kind_of? FalseClass + @@FALSE_CODE + elsif v.kind_of? SingleFloat + @@FLOAT_CODE + elsif v.kind_of? Float + @@DOUBLE_CODE + elsif v.kind_of? UUID + @@UUID_CODE + elsif v.kind_of? Array + @@NESTED_CODE + else + raise ArgumentError, "unsupported type #{v.class}" + end + end + + def self._compare_floats(f1, f2, is_double) + # This converts the floats to their byte representation and then + # does the comparison. Why? + # 1) NaN comparison - Ruby doesn't really do this + # 2) -0.0 == 0.0 in Ruby but not in our representation + # It would be better to just take the floats and compare them, but + # this way handles the edge cases correctly. + b1 = float_adjust([f1].pack(is_double ? ">G" : ">g"), 0, (is_double ? 8 : 4), true) + b2 = float_adjust([f2].pack(is_double ? ">G" : ">g"), 0, (is_double ? 8 : 4), true) + b1 <=> b2 + end + + def self._compare_elems(v1, v2) + c1 = _code_for(v1) + c2 = _code_for(v2) + return c1 <=> c2 unless c1 == c2 + + if c1 == @@NULL_CODE + 0 + elsif c1 == @@DOUBLE_CODE + _compare_floats(v1, v2, true) + elsif c1 == @@NESTED_CODE + compare(v1, v2) # recurse + else + v1 <=> v2 + end + end + + def self.compare(tuple1, tuple2) + i = 0 + while i < tuple1.length && i < tuple2.length + c = self._compare_elems(tuple1[i], tuple2[i]) + return c unless c == 0 + i += 1 + end + tuple1.length <=> tuple2.length + end end end diff --git a/bindings/ruby/tests/tester.rb b/bindings/ruby/tests/tester.rb index f67634158f..33b7c073d7 100755 --- a/bindings/ruby/tests/tester.rb +++ b/bindings/ruby/tests/tester.rb @@ -393,6 +393,27 @@ class Tester (FDB::Tuple.range arr).each do |x| inst.push(x) end + when "TUPLE_SORT" + arr = [] + inst.wait_and_pop.times do |i| + arr.push(FDB::Tuple.unpack inst.wait_and_pop) + end + arr.sort! { |t1, t2| FDB::Tuple.compare(t1, t2) } + arr.each do |x| + inst.push( FDB::Tuple.pack x) + end + when "ENCODE_FLOAT" + bytes = inst.wait_and_pop + inst.push(FDB::Tuple::SingleFloat.new(bytes.unpack("g")[0])) + when "ENCODE_DOUBLE" + bytes = inst.wait_and_pop + inst.push(bytes.unpack("G")[0]) + when "DECODE_FLOAT" + f_val = inst.wait_and_pop + inst.push([f_val.value].pack("g")) + when "DECODE_DOUBLE" + d_val = inst.wait_and_pop + inst.push([d_val].pack("G")) when "START_THREAD" t = Tester.new( @db, inst.wait_and_pop ) thr = Thread.new do diff --git a/flow/error_definitions.h b/flow/error_definitions.h index daebd6947d..46246d1f43 100644 --- a/flow/error_definitions.h +++ b/flow/error_definitions.h @@ -157,6 +157,7 @@ ERROR( directory_prefix_not_empty, 2264, "The database has keys stored at the pr ERROR( directory_prefix_in_use, 2265, "The directory layer already has a conflicting prefix" ) ERROR( invalid_destination_directory, 2266, "The target directory is invalid" ) ERROR( cannot_modify_root_directory, 2267, "The root directory cannot be modified" ) +ERROR( invalid_uuid_size, 2268, "UUID is not sixteen bytes"); // 2300 - backup and restore errors ERROR( backup_error, 2300, "Backup error")