diff --git a/bindings/c/fdb_c.cpp b/bindings/c/fdb_c.cpp index f762b605b2..a5d6a206d8 100644 --- a/bindings/c/fdb_c.cpp +++ b/bindings/c/fdb_c.cpp @@ -459,10 +459,8 @@ FDBFuture* fdb_transaction_get_range_impl(FDBTransaction* tr, const int mode_bytes_array[] = { GetRangeLimits::BYTE_LIMIT_UNLIMITED, 256, 1000, 4096, 80000 }; /* The progression used for FDB_STREAMING_MODE_ITERATOR. - Goes from small -> medium -> large. Then 1.5 * previous until serial. */ - static const int iteration_progression[] = { - 256, 1000, 4096, 6144, 9216, 13824, 20736, 31104, 46656, 69984, 80000 - }; + Goes 1.5 * previous. */ + static const int iteration_progression[] = { 4096, 6144, 9216, 13824, 20736, 31104, 46656, 69984, 80000, 120000 }; /* length(iteration_progression) */ static const int max_iteration = sizeof(iteration_progression) / sizeof(int); diff --git a/bindings/java/CMakeLists.txt b/bindings/java/CMakeLists.txt index 1ec83d46f6..c72078adc5 100644 --- a/bindings/java/CMakeLists.txt +++ b/bindings/java/CMakeLists.txt @@ -59,6 +59,8 @@ set(JAVA_BINDING_SRCS src/main/com/apple/foundationdb/subspace/Subspace.java src/main/com/apple/foundationdb/Transaction.java src/main/com/apple/foundationdb/TransactionContext.java + src/main/com/apple/foundationdb/EventKeeper.java + src/main/com/apple/foundationdb/MapEventKeeper.java src/main/com/apple/foundationdb/testing/AbstractWorkload.java src/main/com/apple/foundationdb/testing/WorkloadContext.java src/main/com/apple/foundationdb/testing/Promise.java diff --git a/bindings/java/src/integration/com/apple/foundationdb/RangeQueryIntegrationTest.java b/bindings/java/src/integration/com/apple/foundationdb/RangeQueryIntegrationTest.java index 4ab594463c..e7490fd038 100644 --- a/bindings/java/src/integration/com/apple/foundationdb/RangeQueryIntegrationTest.java +++ b/bindings/java/src/integration/com/apple/foundationdb/RangeQueryIntegrationTest.java @@ -21,6 +21,8 @@ package com.apple.foundationdb; import java.util.Iterator; import java.util.Map; +import java.util.NavigableMap; +import java.util.Random; import java.util.TreeMap; import com.apple.foundationdb.async.AsyncIterable; @@ -55,6 +57,48 @@ class RangeQueryIntegrationTest { } } + private void loadData(Database db, Map dataToLoad) { + db.run(tr -> { + for (Map.Entry entry : dataToLoad.entrySet()) { + tr.set(entry.getKey(), entry.getValue()); + } + return null; + }); + } + + @Test + public void canGetRowWithKeySelector() throws Exception { + Random rand = new Random(); + byte[] key = new byte[128]; + byte[] value = new byte[128]; + rand.nextBytes(key); + key[0] = (byte)0xEE; + rand.nextBytes(value); + + NavigableMap data = new TreeMap<>(ByteArrayUtil.comparator()); + data.put(key, value); + try (Database db = fdb.open()) { + loadData(db, data); + db.run(tr -> { + byte[] actualValue = tr.get(key).join(); + Assertions.assertNotNull(actualValue, "Missing key!"); + Assertions.assertArrayEquals(value, actualValue, "incorrect value!"); + + KeySelector start = KeySelector.firstGreaterOrEqual(new byte[] { key[0] }); + KeySelector end = KeySelector.firstGreaterOrEqual(ByteArrayUtil.strinc(start.getKey())); + AsyncIterable kvIterable = tr.getRange(start, end); + AsyncIterator kvs = kvIterable.iterator(); + + Assertions.assertTrue(kvs.hasNext(), "Did not return a record!"); + KeyValue n = kvs.next(); + Assertions.assertArrayEquals(key, n.getKey(), "Did not return a key correctly!"); + Assertions.assertArrayEquals(value, n.getValue(), "Did not return the corect value!"); + + return null; + }); + } + } + @Test void rangeQueryReturnsResults() throws Exception { /* diff --git a/bindings/java/src/junit/com/apple/foundationdb/EventKeeperTest.java b/bindings/java/src/junit/com/apple/foundationdb/EventKeeperTest.java new file mode 100644 index 0000000000..407bee2518 --- /dev/null +++ b/bindings/java/src/junit/com/apple/foundationdb/EventKeeperTest.java @@ -0,0 +1,123 @@ +/* + * EventKeeperTest.java + * + * This source file is part of the FoundationDB open source project + * + * Copyright 2013-2021 Apple Inc. and the FoundationDB project authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.apple.foundationdb; + +import java.util.ArrayList; +import java.util.Arrays; +import java.util.HashMap; +import java.util.List; +import java.util.Map; + +import com.apple.foundationdb.EventKeeper.Events; +import com.apple.foundationdb.async.AsyncIterator; +import com.apple.foundationdb.tuple.ByteArrayUtil; + +import org.junit.jupiter.api.Assertions; +import org.junit.jupiter.api.Test; + +/** + * Basic test code for testing basic Transaction Timer logic. + * + * These tests don't check for a whole lot, they just verify that + * instrumentation works as expected for specific patterns. + */ +class EventKeeperTest { + + @Test + void testSetVersion() throws Exception { + + EventKeeper timer = new MapEventKeeper(); + + try (FDBTransaction txn = new FDBTransaction(1, null, null, timer)) { + Assertions.assertThrows(UnsatisfiedLinkError.class, + () -> { txn.setReadVersion(1L); }, "Test should call a bad native method"); + long jniCalls = timer.getCount(Events.JNI_CALL); + + Assertions.assertEquals(1L, jniCalls, "Unexpected number of JNI calls:"); + }catch(UnsatisfiedLinkError ignored){ + //this is necessary to prevent an exception being thrown at close time + } + } + + @Test + void testGetReadVersion() throws Exception { + EventKeeper timer = new MapEventKeeper(); + + try (FDBTransaction txn = new FDBTransaction(1, null, null, timer)) { + Assertions.assertThrows(UnsatisfiedLinkError.class, + () -> { txn.getReadVersion(); }, "Test should call a bad native method"); + long jniCalls = timer.getCount(Events.JNI_CALL); + + Assertions.assertEquals(1L, jniCalls, "Unexpected number of JNI calls:"); + }catch(UnsatisfiedLinkError ignored){ + //required to prevent an extra exception being thrown at close time + } + } + + @Test + void testGetRangeRecordsFetches() throws Exception { + EventKeeper timer = new MapEventKeeper(); + List testKvs = Arrays.asList(new KeyValue("hello".getBytes(), "goodbye".getBytes())); + + FDBTransaction txn = new FakeFDBTransaction(testKvs, 1L, null, null); + + RangeQuery query = new RangeQuery(txn, true, KeySelector.firstGreaterOrEqual(new byte[] { 0x00 }), + KeySelector.firstGreaterOrEqual(new byte[] { (byte)0xFF }), -1, false, + StreamingMode.ITERATOR, timer); + AsyncIterator iter = query.iterator(); + + List iteratedItems = new ArrayList<>(); + while (iter.hasNext()) { + iteratedItems.add(iter.next()); + } + + // basic verification that we got back what we expected to get back. + Assertions.assertEquals(testKvs.size(), iteratedItems.size(), "Incorrect iterated list, size incorrect."); + + int expectedByteSize = 0; + for (KeyValue expected : testKvs) { + byte[] eKey = expected.getKey(); + byte[] eVal = expected.getValue(); + expectedByteSize += eKey.length + 4; + expectedByteSize += eVal.length + 4; + boolean found = false; + for (KeyValue actual : iteratedItems) { + byte[] aKey = actual.getKey(); + byte[] aVal = actual.getValue(); + if (ByteArrayUtil.compareTo(eKey, 0, eKey.length, aKey, 0, aKey.length) == 0) { + int cmp = ByteArrayUtil.compareTo(eVal, 0, eVal.length, aVal, 0, aVal.length); + Assertions.assertEquals(0, cmp, "Incorrect value returned"); + found = true; + break; + } + } + + Assertions.assertTrue(found, "missing key!"); + } + + // now check the timer and see if it recorded any events + Assertions.assertEquals(1, timer.getCount(Events.RANGE_QUERY_FETCHES), "Unexpected number of chunk fetches"); + Assertions.assertEquals(testKvs.size(), timer.getCount(Events.RANGE_QUERY_RECORDS_FETCHED), + "Unexpected number of tuples fetched"); + Assertions.assertEquals(expectedByteSize, timer.getCount(Events.BYTES_FETCHED), + "Incorrect number of bytes fetched"); + } + +} diff --git a/bindings/java/src/junit/com/apple/foundationdb/FakeFDBTransaction.java b/bindings/java/src/junit/com/apple/foundationdb/FakeFDBTransaction.java index 6ecb4b44f7..f154790b2b 100644 --- a/bindings/java/src/junit/com/apple/foundationdb/FakeFDBTransaction.java +++ b/bindings/java/src/junit/com/apple/foundationdb/FakeFDBTransaction.java @@ -67,6 +67,15 @@ public class FakeFDBTransaction extends FDBTransaction { } } + public FakeFDBTransaction(List backingData, long cPtr, Database db, + Executor executor) { + this(cPtr, db, executor); + + for (KeyValue entry : backingData) { + this.backingData.put(entry.getKey(), entry.getValue()); + } + } + @Override public CompletableFuture get(byte[] key) { return CompletableFuture.completedFuture(this.backingData.get(key)); @@ -92,7 +101,7 @@ public class FakeFDBTransaction extends FDBTransaction { // holder variable so that we can pass the range to the results function safely final NavigableMap retMap = range; - FutureResults fr = new FutureResults(-1L, false, executor) { + FutureResults fr = new FutureResults(-1L, false, executor, null) { @Override protected void registerMarshalCallback(Executor executor) { // no-op diff --git a/bindings/java/src/main/com/apple/foundationdb/Database.java b/bindings/java/src/main/com/apple/foundationdb/Database.java index 86e06e42a9..e5f2d36de6 100644 --- a/bindings/java/src/main/com/apple/foundationdb/Database.java +++ b/bindings/java/src/main/com/apple/foundationdb/Database.java @@ -62,6 +62,17 @@ public interface Database extends AutoCloseable, TransactionContext { */ Transaction createTransaction(Executor e); + /** + * Creates a {@link Transaction} that operates on this {@code Database} with the given {@link Executor} + * for asynchronous callbacks. + * + * @param e the {@link Executor} to use when executing asynchronous callbacks for the database + * @param eventKeeper the {@link EventKeeper} to use when tracking instrumented calls for the transaction. + * + * @return a newly created {@code Transaction} that reads from and writes to this {@code Database}. + */ + Transaction createTransaction(Executor e, EventKeeper eventKeeper); + /** * Returns a set of options that can be set on a {@code Database} * diff --git a/bindings/java/src/main/com/apple/foundationdb/EventKeeper.java b/bindings/java/src/main/com/apple/foundationdb/EventKeeper.java new file mode 100644 index 0000000000..04d66f0f50 --- /dev/null +++ b/bindings/java/src/main/com/apple/foundationdb/EventKeeper.java @@ -0,0 +1,172 @@ +/* + * EventKeeper.java + * + * This source file is part of the FoundationDB open source project + * + * Copyright 2013-2021 Apple Inc. and the FoundationDB project authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.apple.foundationdb; + +import java.util.concurrent.TimeUnit; + +/** + * A device for externally instrumenting the FDB java driver, for monitoring + * purposes. + * + * Note that implementations as expected to be thread-safe, and may be manipulated + * from multiple threads even in nicely single-threaded-looking applications. + */ +public interface EventKeeper { + + /** + * Count the number of events which occurred. + * + * @param event the event which occurred + * @param amt the number of times that even occurred + */ + void count(Event event, long amt); + + /** + * Convenience method to add 1 to the number of events which occurred. + * + * @param event the event which occurred. + */ + default void increment(Event event) { count(event, 1L); } + + /** + * Count the time taken to perform an event, in nanoseconds. + * + * Note that {@code event.isTimeEvent()} should return true here. + * + * @param event the event which was timed (the event should be a time event). + * @param nanos the amount of time taken (in nanoseconds) + */ + void timeNanos(Event event, long nanos); + + /** + * Count the time taken to perform an action, in the specified units. + * + * Note that {@code event.isTimeEvent()} should return true. + * + * @param event the event which was timed. + * @param duration the time taken + * @param theUnit the unit of time in which the time measurement was taken + */ + default void time(Event event, long duration, TimeUnit theUnit) { timeNanos(event, theUnit.toNanos(duration)); } + + /** + * Get the number of events which occurred since this timer was created. + * + * If the event was never recorded, then this returns 0. + * + * @param event the event to get the count for + * @return the number of times the event was triggered. If the event has never been triggered, + * then this returns 0 + */ + long getCount(Event event); + + /** + * Get the amount of time taken by this event, in nanoseconds. + * + * @param event the event to get the time for + * @return the total time measured for this event, in nanoseconds. If the event was never recorded, + * return 0 instead. + */ + long getTimeNanos(Event event); + + /** + * Get the amount of time taken by this event, in the specified units. + * + * Important note: If the time that was measured in nanoseconds does not evenly divide the unit that + * is specified (which is likely, considering time), then some precision may be lost in the conversion. Use + * this carefully. + * + * @param event the event to get the time for + * @param theUnit the unit to get time in + * @return the total time measured for this event, in the specified unit. If the event was never recorded, + * return 0. + */ + default long getTime(Event event, TimeUnit theUnit) { + return theUnit.convert(getTimeNanos(event), TimeUnit.NANOSECONDS); + } + + /** + * Marker interface for tracking the specific type of event that occurs, and metadata about said event. + * + * Implementations should be sure to provide a quality {@code equals} and {@code hashCode}. + */ + interface Event { + /** + * @return the name of this event, as a unique string. This name should generally be unique, because + * it's likely that {@code EventKeeper} implementations will rely on this for uniqueness. + */ + String name(); + + /** + * @return true if this event represents a timed event, rather than a counter event. + */ + default boolean isTimeEvent() { return false; }; + } + + /** + * An enumeration of static events which occur within the FDB Java driver. + */ + enum Events implements Event { + /** + * The number of JNI calls that were exercised. + */ + JNI_CALL, + + /** + * The total number of bytes pulled from the native layer, including length delimiters., from + * {@link Transaction#get(byte[])}, {@link Transaction#getKey(KeySelector)}, + * {@link Transaction#getRange(KeySelector, KeySelector)} (and related method + * overrides), or any other read-type operation which occurs on a Transaction + */ + BYTES_FETCHED, + + /** + * The number of times a DirectBuffer was used to transfer a range query chunk + * across the JNI boundary + */ + RANGE_QUERY_DIRECT_BUFFER_HIT, + /** + * The number of times a range query chunk was unable to use a DirectBuffer to + * transfer data across the JNI boundary + */ + RANGE_QUERY_DIRECT_BUFFER_MISS, + /** + * The number of direct fetches made during a range query + */ + RANGE_QUERY_FETCHES, + /** + * The number of tuples fetched during a range query + */ + RANGE_QUERY_RECORDS_FETCHED, + /** + * The number of times a range query chunk fetch failed + */ + RANGE_QUERY_CHUNK_FAILED, + /** + * The time taken to perform an internal `getRange` fetch, in nanoseconds + */ + RANGE_QUERY_FETCH_TIME_NANOS { + @Override + public boolean isTimeEvent() { + return true; + } + }; + } +} diff --git a/bindings/java/src/main/com/apple/foundationdb/FDB.java b/bindings/java/src/main/com/apple/foundationdb/FDB.java index 4cab50c92b..031a1e2472 100644 --- a/bindings/java/src/main/com/apple/foundationdb/FDB.java +++ b/bindings/java/src/main/com/apple/foundationdb/FDB.java @@ -364,6 +364,28 @@ public class FDB { return open(clusterFilePath, DEFAULT_EXECUTOR); } + /** + * Initializes networking if required and connects to the cluster specified by {@code clusterFilePath}.
+ *
+ * A single client can use this function multiple times to connect to different + * clusters simultaneously, with each invocation requiring its own cluster file. + * To connect to multiple clusters running at different, incompatible versions, + * the multi-version client API + * must be used. + * + * @param clusterFilePath the + * cluster file + * defining the FoundationDB cluster. This can be {@code null} if the + * default fdb.cluster file + * is to be used. + * @param eventKeeper the EventKeeper to use for instrumentation calls, or {@code null} if no instrumentation is desired. + * + * @return a {@code CompletableFuture} that will be set to a FoundationDB {@link Database} + */ + public Database open(String clusterFilePath, EventKeeper eventKeeper) throws FDBException { + return open(clusterFilePath, DEFAULT_EXECUTOR, eventKeeper); + } + /** * Initializes networking if required and connects to the cluster specified by {@code clusterFilePath}.
*
@@ -383,13 +405,37 @@ public class FDB { * @return a {@code CompletableFuture} that will be set to a FoundationDB {@link Database} */ public Database open(String clusterFilePath, Executor e) throws FDBException { + return open(clusterFilePath, e, null); + } + + /** + * Initializes networking if required and connects to the cluster specified by {@code clusterFilePath}.
+ *
+ * A single client can use this function multiple times to connect to different + * clusters simultaneously, with each invocation requiring its own cluster file. + * To connect to multiple clusters running at different, incompatible versions, + * the multi-version client API + * must be used. + * + * @param clusterFilePath the + * cluster file + * defining the FoundationDB cluster. This can be {@code null} if the + * default fdb.cluster file + * is to be used. + * @param e the {@link Executor} to use to execute asynchronous callbacks + * @param eventKeeper the {@link EventKeeper} to use to record instrumentation metrics, or {@code null} if no + * instrumentation is desired. + * + * @return a {@code CompletableFuture} that will be set to a FoundationDB {@link Database} + */ + public Database open(String clusterFilePath, Executor e, EventKeeper eventKeeper) throws FDBException { synchronized(this) { if(!isConnected()) { startNetwork(); } } - return new FDBDatabase(Database_create(clusterFilePath), e); + return new FDBDatabase(Database_create(clusterFilePath), e, eventKeeper); } /** diff --git a/bindings/java/src/main/com/apple/foundationdb/FDBDatabase.java b/bindings/java/src/main/com/apple/foundationdb/FDBDatabase.java index 038aacb2e6..620b5aaa4e 100644 --- a/bindings/java/src/main/com/apple/foundationdb/FDBDatabase.java +++ b/bindings/java/src/main/com/apple/foundationdb/FDBDatabase.java @@ -31,11 +31,17 @@ import com.apple.foundationdb.async.AsyncUtil; class FDBDatabase extends NativeObjectWrapper implements Database, OptionConsumer { private DatabaseOptions options; private final Executor executor; + private final EventKeeper eventKeeper; protected FDBDatabase(long cPtr, Executor executor) { + this(cPtr, executor, null); + } + + protected FDBDatabase(long cPtr, Executor executor, EventKeeper eventKeeper) { super(cPtr); this.executor = executor; this.options = new DatabaseOptions(this); + this.eventKeeper = eventKeeper; } @Override @@ -112,14 +118,19 @@ class FDBDatabase extends NativeObjectWrapper implements Database, OptionConsume @Override public Transaction createTransaction(Executor e) { + return createTransaction(e, eventKeeper); + } + + @Override + public Transaction createTransaction(Executor e, EventKeeper eventKeeper) { pointerReadLock.lock(); Transaction tr = null; try { - tr = new FDBTransaction(Database_createTransaction(getPtr()), this, e); + tr = new FDBTransaction(Database_createTransaction(getPtr()), this, e, eventKeeper); tr.options().setUsedDuringCommitProtectionDisable(); return tr; - } catch(RuntimeException err) { - if(tr != null) { + } catch (RuntimeException err) { + if (tr != null) { tr.close(); } diff --git a/bindings/java/src/main/com/apple/foundationdb/FDBTransaction.java b/bindings/java/src/main/com/apple/foundationdb/FDBTransaction.java index a01edc00a8..05431a0fba 100644 --- a/bindings/java/src/main/com/apple/foundationdb/FDBTransaction.java +++ b/bindings/java/src/main/com/apple/foundationdb/FDBTransaction.java @@ -26,6 +26,7 @@ import java.util.concurrent.ExecutionException; import java.util.concurrent.Executor; import java.util.function.Function; +import com.apple.foundationdb.EventKeeper.Events; import com.apple.foundationdb.async.AsyncIterable; import com.apple.foundationdb.async.AsyncUtil; import com.apple.foundationdb.tuple.ByteArrayUtil; @@ -34,6 +35,7 @@ class FDBTransaction extends NativeObjectWrapper implements Transaction, OptionC private final Database database; private final Executor executor; private final TransactionOptions options; + private final EventKeeper eventKeeper; private boolean transactionOwner; public final ReadTransaction snapshot; @@ -93,9 +95,9 @@ class FDBTransaction extends NativeObjectWrapper implements Transaction, OptionC // getRange -> KeySelectors /////////////////// @Override - public AsyncIterable getRange(KeySelector begin, KeySelector end, - int limit, boolean reverse, StreamingMode mode) { - return new RangeQuery(FDBTransaction.this, true, begin, end, limit, reverse, mode); + public AsyncIterable getRange(KeySelector begin, KeySelector end, int limit, boolean reverse, + StreamingMode mode) { + return new RangeQuery(FDBTransaction.this, true, begin, end, limit, reverse, mode, eventKeeper); } @Override public AsyncIterable getRange(KeySelector begin, KeySelector end, @@ -195,9 +197,15 @@ class FDBTransaction extends NativeObjectWrapper implements Transaction, OptionC } protected FDBTransaction(long cPtr, Database database, Executor executor) { + //added for backwards compatibility with subclasses contained in different projects + this(cPtr,database,executor,null); + } + + protected FDBTransaction(long cPtr, Database database, Executor executor, EventKeeper eventKeeper) { super(cPtr); this.database = database; this.executor = executor; + this.eventKeeper = eventKeeper; snapshot = new ReadSnapshot(); options = new TransactionOptions(this); transactionOwner = true; @@ -220,6 +228,17 @@ class FDBTransaction extends NativeObjectWrapper implements Transaction, OptionC @Override public void setReadVersion(long version) { + /* + * Note that this is done outside of the lock, because we don't want to rely on + * the caller code being particularly efficient, and if we get a bad + * implementation of a eventKeeper, we could end up holding the pointerReadLock for an + * arbitrary amount of time; this would be Bad(TM), so we execute this outside + * the lock, so that in the worst case only the caller thread itself can be hurt + * by bad callbacks. + */ + if (eventKeeper != null) { + eventKeeper.increment(Events.JNI_CALL); + } pointerReadLock.lock(); try { Transaction_setVersion(getPtr(), version); @@ -233,6 +252,9 @@ class FDBTransaction extends NativeObjectWrapper implements Transaction, OptionC */ @Override public CompletableFuture getReadVersion() { + if (eventKeeper != null) { + eventKeeper.increment(Events.JNI_CALL); + } pointerReadLock.lock(); try { return new FutureInt64(Transaction_getReadVersion(getPtr()), executor); @@ -250,9 +272,12 @@ class FDBTransaction extends NativeObjectWrapper implements Transaction, OptionC } private CompletableFuture get_internal(byte[] key, boolean isSnapshot) { + if (eventKeeper != null) { + eventKeeper.increment(Events.JNI_CALL); + } pointerReadLock.lock(); try { - return new FutureResult(Transaction_get(getPtr(), key, isSnapshot), executor); + return new FutureResult(Transaction_get(getPtr(), key, isSnapshot), executor,eventKeeper); } finally { pointerReadLock.unlock(); } @@ -267,10 +292,14 @@ class FDBTransaction extends NativeObjectWrapper implements Transaction, OptionC } private CompletableFuture getKey_internal(KeySelector selector, boolean isSnapshot) { + if (eventKeeper != null) { + eventKeeper.increment(Events.JNI_CALL); + } pointerReadLock.lock(); try { - return new FutureKey(Transaction_getKey(getPtr(), - selector.getKey(), selector.orEqual(), selector.getOffset(), isSnapshot), executor); + return new FutureKey( + Transaction_getKey(getPtr(), selector.getKey(), selector.orEqual(), selector.getOffset(), isSnapshot), + executor, eventKeeper); } finally { pointerReadLock.unlock(); } @@ -278,6 +307,9 @@ class FDBTransaction extends NativeObjectWrapper implements Transaction, OptionC @Override public CompletableFuture getEstimatedRangeSizeBytes(byte[] begin, byte[] end) { + if (eventKeeper != null) { + eventKeeper.increment(Events.JNI_CALL); + } pointerReadLock.lock(); try { return new FutureInt64(Transaction_getEstimatedRangeSizeBytes(getPtr(), begin, end), executor); @@ -312,7 +344,7 @@ class FDBTransaction extends NativeObjectWrapper implements Transaction, OptionC @Override public AsyncIterable getRange(KeySelector begin, KeySelector end, int limit, boolean reverse, StreamingMode mode) { - return new RangeQuery(this, false, begin, end, limit, reverse, mode); + return new RangeQuery(this, false, begin, end, limit, reverse, mode, eventKeeper); } @Override public AsyncIterable getRange(KeySelector begin, KeySelector end, @@ -384,9 +416,12 @@ class FDBTransaction extends NativeObjectWrapper implements Transaction, OptionC // Users of this function must close the returned FutureResults when finished protected FutureResults getRange_internal( - KeySelector begin, KeySelector end, - int rowLimit, int targetBytes, int streamingMode, - int iteration, boolean isSnapshot, boolean reverse) { + KeySelector begin, KeySelector end, + int rowLimit, int targetBytes, int streamingMode, + int iteration, boolean isSnapshot, boolean reverse) { + if (eventKeeper != null) { + eventKeeper.increment(Events.JNI_CALL); + } pointerReadLock.lock(); try { /*System.out.println(String.format( @@ -397,7 +432,7 @@ class FDBTransaction extends NativeObjectWrapper implements Transaction, OptionC Transaction_getRange(getPtr(), begin.getKey(), begin.orEqual(), begin.getOffset(), end.getKey(), end.orEqual(), end.getOffset(), rowLimit, targetBytes, streamingMode, iteration, isSnapshot, reverse), - FDB.instance().isDirectBufferQueriesEnabled(), executor); + FDB.instance().isDirectBufferQueriesEnabled(), executor, eventKeeper); } finally { pointerReadLock.unlock(); } @@ -435,8 +470,10 @@ class FDBTransaction extends NativeObjectWrapper implements Transaction, OptionC addConflictRange(key, ByteArrayUtil.join(key, new byte[] { (byte)0 }), ConflictRangeType.WRITE); } - private void addConflictRange(byte[] keyBegin, byte[] keyEnd, - ConflictRangeType type) { + private void addConflictRange(byte[] keyBegin, byte[] keyEnd, ConflictRangeType type) { + if (eventKeeper != null) { + eventKeeper.increment(Events.JNI_CALL); + } pointerReadLock.lock(); try { Transaction_addConflictRange(getPtr(), keyBegin, keyEnd, type.code()); @@ -469,8 +506,11 @@ class FDBTransaction extends NativeObjectWrapper implements Transaction, OptionC @Override public void set(byte[] key, byte[] value) { - if(key == null || value == null) + if (key == null || value == null) throw new IllegalArgumentException("Keys/Values must be non-null"); + if (eventKeeper != null) { + eventKeeper.increment(Events.JNI_CALL); + } pointerReadLock.lock(); try { Transaction_set(getPtr(), key, value); @@ -481,8 +521,11 @@ class FDBTransaction extends NativeObjectWrapper implements Transaction, OptionC @Override public void clear(byte[] key) { - if(key == null) + if (key == null) throw new IllegalArgumentException("Key cannot be null"); + if (eventKeeper != null) { + eventKeeper.increment(Events.JNI_CALL); + } pointerReadLock.lock(); try { Transaction_clear(getPtr(), key); @@ -493,8 +536,11 @@ class FDBTransaction extends NativeObjectWrapper implements Transaction, OptionC @Override public void clear(byte[] beginKey, byte[] endKey) { - if(beginKey == null || endKey == null) + if (beginKey == null || endKey == null) throw new IllegalArgumentException("Keys cannot be null"); + if (eventKeeper != null) { + eventKeeper.increment(Events.JNI_CALL); + } pointerReadLock.lock(); try { Transaction_clear(getPtr(), beginKey, endKey); @@ -516,6 +562,9 @@ class FDBTransaction extends NativeObjectWrapper implements Transaction, OptionC @Override public void mutate(MutationType optype, byte[] key, byte[] value) { + if (eventKeeper != null) { + eventKeeper.increment(Events.JNI_CALL); + } pointerReadLock.lock(); try { Transaction_mutate(getPtr(), optype.code(), key, value); @@ -526,6 +575,9 @@ class FDBTransaction extends NativeObjectWrapper implements Transaction, OptionC @Override public void setOption(int code, byte[] param) { + if (eventKeeper != null) { + eventKeeper.increment(Events.JNI_CALL); + } pointerReadLock.lock(); try { Transaction_setOption(getPtr(), code, param); @@ -536,6 +588,9 @@ class FDBTransaction extends NativeObjectWrapper implements Transaction, OptionC @Override public CompletableFuture commit() { + if (eventKeeper != null) { + eventKeeper.increment(Events.JNI_CALL); + } pointerReadLock.lock(); try { return new FutureVoid(Transaction_commit(getPtr()), executor); @@ -546,6 +601,9 @@ class FDBTransaction extends NativeObjectWrapper implements Transaction, OptionC @Override public Long getCommittedVersion() { + if (eventKeeper != null) { + eventKeeper.increment(Events.JNI_CALL); + } pointerReadLock.lock(); try { return Transaction_getCommittedVersion(getPtr()); @@ -556,9 +614,12 @@ class FDBTransaction extends NativeObjectWrapper implements Transaction, OptionC @Override public CompletableFuture getVersionstamp() { + if (eventKeeper != null) { + eventKeeper.increment(Events.JNI_CALL); + } pointerReadLock.lock(); try { - return new FutureKey(Transaction_getVersionstamp(getPtr()), executor); + return new FutureKey(Transaction_getVersionstamp(getPtr()), executor, eventKeeper); } finally { pointerReadLock.unlock(); } @@ -566,6 +627,9 @@ class FDBTransaction extends NativeObjectWrapper implements Transaction, OptionC @Override public CompletableFuture getApproximateSize() { + if (eventKeeper != null) { + eventKeeper.increment(Events.JNI_CALL); + } pointerReadLock.lock(); try { return new FutureInt64(Transaction_getApproximateSize(getPtr()), executor); @@ -576,6 +640,9 @@ class FDBTransaction extends NativeObjectWrapper implements Transaction, OptionC @Override public CompletableFuture watch(byte[] key) throws FDBException { + if (eventKeeper != null) { + eventKeeper.increment(Events.JNI_CALL); + } pointerReadLock.lock(); try { return new FutureVoid(Transaction_watch(getPtr(), key), executor); @@ -586,24 +653,27 @@ class FDBTransaction extends NativeObjectWrapper implements Transaction, OptionC @Override public CompletableFuture onError(Throwable e) { - if((e instanceof CompletionException || e instanceof ExecutionException) && e.getCause() != null) { + if ((e instanceof CompletionException || e instanceof ExecutionException) && e.getCause() != null) { e = e.getCause(); } - if(!(e instanceof FDBException)) { + if (!(e instanceof FDBException)) { CompletableFuture future = new CompletableFuture<>(); future.completeExceptionally(e); return future; } + if (eventKeeper != null) { + eventKeeper.increment(Events.JNI_CALL); + } pointerReadLock.lock(); try { - CompletableFuture f = new FutureVoid(Transaction_onError(getPtr(), ((FDBException)e).getCode()), executor); + CompletableFuture f = new FutureVoid(Transaction_onError(getPtr(), ((FDBException) e).getCode()), + executor); final Transaction tr = transfer(); - return f.thenApply(v -> tr) - .whenComplete((v, t) -> { - if(t != null) { - tr.close(); - } - }); + return f.thenApply(v -> tr).whenComplete((v, t) -> { + if (t != null) { + tr.close(); + } + }); } finally { pointerReadLock.unlock(); if(!transactionOwner) { @@ -614,6 +684,9 @@ class FDBTransaction extends NativeObjectWrapper implements Transaction, OptionC @Override public void cancel() { + if(eventKeeper!=null){ + eventKeeper.increment(Events.JNI_CALL); + } pointerReadLock.lock(); try { Transaction_cancel(getPtr()); @@ -623,6 +696,9 @@ class FDBTransaction extends NativeObjectWrapper implements Transaction, OptionC } public CompletableFuture getAddressesForKey(byte[] key) { + if(eventKeeper!=null){ + eventKeeper.increment(Events.JNI_CALL); + } pointerReadLock.lock(); try { return new FutureStrings(Transaction_getKeyLocations(getPtr(), key), executor); @@ -672,6 +748,9 @@ class FDBTransaction extends NativeObjectWrapper implements Transaction, OptionC @Override protected void closeInternal(long cPtr) { + if(eventKeeper!=null){ + eventKeeper.increment(Events.JNI_CALL); + } if(transactionOwner) { Transaction_dispose(cPtr); } diff --git a/bindings/java/src/main/com/apple/foundationdb/FutureKey.java b/bindings/java/src/main/com/apple/foundationdb/FutureKey.java index bd101500fd..d959267bc6 100644 --- a/bindings/java/src/main/com/apple/foundationdb/FutureKey.java +++ b/bindings/java/src/main/com/apple/foundationdb/FutureKey.java @@ -22,9 +22,13 @@ package com.apple.foundationdb; import java.util.concurrent.Executor; +import com.apple.foundationdb.EventKeeper.Events; + class FutureKey extends NativeFuture { - FutureKey(long cPtr, Executor executor) { + private final EventKeeper eventKeeper; + FutureKey(long cPtr, Executor executor, EventKeeper eventKeeper) { super(cPtr); + this.eventKeeper = eventKeeper; registerMarshalCallback(executor); } @@ -32,6 +36,14 @@ class FutureKey extends NativeFuture { protected byte[] getIfDone_internal(long cPtr) throws FDBException { return FutureKey_get(cPtr); } + + @Override + protected void postMarshal(byte[] value) { + if(value!=null && eventKeeper!=null){ + eventKeeper.count(Events.BYTES_FETCHED, value.length); + } + super.postMarshal(value); + } private native byte[] FutureKey_get(long cPtr) throws FDBException; } diff --git a/bindings/java/src/main/com/apple/foundationdb/FutureResult.java b/bindings/java/src/main/com/apple/foundationdb/FutureResult.java index 19ae315b1f..824ae07e74 100644 --- a/bindings/java/src/main/com/apple/foundationdb/FutureResult.java +++ b/bindings/java/src/main/com/apple/foundationdb/FutureResult.java @@ -22,9 +22,14 @@ package com.apple.foundationdb; import java.util.concurrent.Executor; +import com.apple.foundationdb.EventKeeper.Events; + class FutureResult extends NativeFuture { - FutureResult(long cPtr, Executor executor) { + private final EventKeeper eventKeeper; + + FutureResult(long cPtr, Executor executor, EventKeeper eventKeeper) { super(cPtr); + this.eventKeeper = eventKeeper; registerMarshalCallback(executor); } @@ -33,5 +38,13 @@ class FutureResult extends NativeFuture { return FutureResult_get(cPtr); } + @Override + protected void postMarshal(byte[] value){ + if(value!=null && eventKeeper!=null){ + eventKeeper.count(Events.BYTES_FETCHED, value.length); + } + super.postMarshal(value); + } + private native byte[] FutureResult_get(long cPtr) throws FDBException; } diff --git a/bindings/java/src/main/com/apple/foundationdb/FutureResults.java b/bindings/java/src/main/com/apple/foundationdb/FutureResults.java index 66461ed1e1..fafde08716 100644 --- a/bindings/java/src/main/com/apple/foundationdb/FutureResults.java +++ b/bindings/java/src/main/com/apple/foundationdb/FutureResults.java @@ -23,20 +23,27 @@ package com.apple.foundationdb; import java.nio.ByteBuffer; import java.util.concurrent.Executor; +import com.apple.foundationdb.EventKeeper.Events; + class FutureResults extends NativeFuture { - FutureResults(long cPtr, boolean enableDirectBufferQueries, Executor executor) { + private final EventKeeper eventKeeper; + FutureResults(long cPtr, boolean enableDirectBufferQueries, Executor executor, EventKeeper eventKeeper) { super(cPtr); registerMarshalCallback(executor); this.enableDirectBufferQueries = enableDirectBufferQueries; + this.eventKeeper = eventKeeper; } @Override - protected void postMarshal() { + protected void postMarshal(RangeResultInfo rri) { // We can't close because this class actually marshals on-demand } @Override protected RangeResultInfo getIfDone_internal(long cPtr) throws FDBException { + if (eventKeeper != null) { + eventKeeper.increment(Events.JNI_CALL); + } FDBException err = Future_getError(cPtr); if(err != null && !err.isSuccess()) { @@ -47,9 +54,15 @@ class FutureResults extends NativeFuture { } public RangeResult getResults() { - ByteBuffer buffer = enableDirectBufferQueries - ? DirectBufferPool.getInstance().poll() - : null; + ByteBuffer buffer = enableDirectBufferQueries ? DirectBufferPool.getInstance().poll() : null; + if (buffer != null && eventKeeper != null) { + eventKeeper.increment(Events.RANGE_QUERY_DIRECT_BUFFER_HIT); + eventKeeper.increment(Events.JNI_CALL); + } else if (eventKeeper != null) { + eventKeeper.increment(Events.RANGE_QUERY_DIRECT_BUFFER_MISS); + eventKeeper.increment(Events.JNI_CALL); + } + try { pointerReadLock.lock(); if (buffer != null) { @@ -68,6 +81,6 @@ class FutureResults extends NativeFuture { private boolean enableDirectBufferQueries = false; private native RangeResult FutureResults_get(long cPtr) throws FDBException; - private native void FutureResults_getDirect(long cPtr, ByteBuffer buffer, int capacity) + private native boolean FutureResults_getDirect(long cPtr, ByteBuffer buffer, int capacity) throws FDBException; } diff --git a/bindings/java/src/main/com/apple/foundationdb/MapEventKeeper.java b/bindings/java/src/main/com/apple/foundationdb/MapEventKeeper.java new file mode 100644 index 0000000000..1232d95e13 --- /dev/null +++ b/bindings/java/src/main/com/apple/foundationdb/MapEventKeeper.java @@ -0,0 +1,72 @@ +/* + * MapEventKeeper.java + * + * This source file is part of the FoundationDB open source project + * + * Copyright 2013-2021 Apple Inc. and the FoundationDB project authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.apple.foundationdb; + +import java.util.concurrent.ConcurrentHashMap; +import java.util.concurrent.ConcurrentMap; +import java.util.concurrent.atomic.AtomicLong; + +/** + * A simple map-based EventKeeper. + * + * This class is thread-safe(per the {@link EventKeeper} spec). It holds all counters in memory; + */ +public class MapEventKeeper implements EventKeeper { + private final ConcurrentMap map = new ConcurrentHashMap<>(); + + @Override + public void count(Event event, long amt) { + Count counter = map.computeIfAbsent(event, (l) -> new Count()); + counter.cnt.addAndGet(amt); + } + + @Override + public void timeNanos(Event event, long nanos) { + Count counter = map.computeIfAbsent(event, (l)->new Count()); + counter.cnt.incrementAndGet(); + counter.duration.addAndGet(nanos); + } + + @Override + public long getCount(Event event) { + Count lng = map.get(event); + if (lng == null) { + return 0L; + } + return lng.cnt.get(); + } + + @Override + public long getTimeNanos(Event event) { + Count lng = map.get(event); + if (lng == null) { + return 0L; + } + return lng.duration.get(); + } + + private static class Count { + private final AtomicLong cnt = new AtomicLong(0L); + + private final AtomicLong duration = new AtomicLong(0L); + + Count(){ } + } +} diff --git a/bindings/java/src/main/com/apple/foundationdb/NativeFuture.java b/bindings/java/src/main/com/apple/foundationdb/NativeFuture.java index 2d22a80617..5c0679cd50 100644 --- a/bindings/java/src/main/com/apple/foundationdb/NativeFuture.java +++ b/bindings/java/src/main/com/apple/foundationdb/NativeFuture.java @@ -54,8 +54,8 @@ abstract class NativeFuture extends CompletableFuture implements AutoClose } private void marshalWhenDone() { + T val = null; try { - T val = null; boolean shouldComplete = false; try { pointerReadLock.lock(); @@ -77,11 +77,11 @@ abstract class NativeFuture extends CompletableFuture implements AutoClose } catch(Throwable t) { completeExceptionally(t); } finally { - postMarshal(); + postMarshal(val); } } - protected void postMarshal() { + protected void postMarshal(T value) { close(); } diff --git a/bindings/java/src/main/com/apple/foundationdb/RangeQuery.java b/bindings/java/src/main/com/apple/foundationdb/RangeQuery.java index f191127c7b..d518a0b9db 100644 --- a/bindings/java/src/main/com/apple/foundationdb/RangeQuery.java +++ b/bindings/java/src/main/com/apple/foundationdb/RangeQuery.java @@ -26,6 +26,7 @@ import java.util.concurrent.CancellationException; import java.util.concurrent.CompletableFuture; import java.util.function.BiConsumer; +import com.apple.foundationdb.EventKeeper.Events; import com.apple.foundationdb.async.AsyncIterable; import com.apple.foundationdb.async.AsyncIterator; import com.apple.foundationdb.async.AsyncUtil; @@ -52,10 +53,10 @@ class RangeQuery implements AsyncIterable { private final int rowLimit; private final boolean reverse; private final StreamingMode streamingMode; + private final EventKeeper eventKeeper; - RangeQuery(FDBTransaction transaction, boolean isSnapshot, - KeySelector begin, KeySelector end, int rowLimit, - boolean reverse, StreamingMode streamingMode) { + RangeQuery(FDBTransaction transaction, boolean isSnapshot, KeySelector begin, KeySelector end, int rowLimit, + boolean reverse, StreamingMode streamingMode, EventKeeper eventKeeper) { this.tr = transaction; this.begin = begin; this.end = end; @@ -63,6 +64,7 @@ class RangeQuery implements AsyncIterable { this.rowLimit = rowLimit; this.reverse = reverse; this.streamingMode = streamingMode; + this.eventKeeper = eventKeeper; } /** @@ -88,9 +90,10 @@ class RangeQuery implements AsyncIterable { .whenComplete((result, e) -> range.close()); } - // If the streaming mode is not EXACT, simply collect the results of an iteration into a list - return AsyncUtil.collect( - new RangeQuery(tr, snapshot, begin, end, rowLimit, reverse, mode), tr.getExecutor()); + // If the streaming mode is not EXACT, simply collect the results of an + // iteration into a list + return AsyncUtil.collect(new RangeQuery(tr, snapshot, begin, end, rowLimit, reverse, mode, eventKeeper), + tr.getExecutor()); } /** @@ -152,9 +155,12 @@ class RangeQuery implements AsyncIterable { @Override public void accept(RangeResultInfo data, Throwable error) { try { - if(error != null) { + if (error != null) { + if (eventKeeper != null) { + eventKeeper.increment(Events.RANGE_QUERY_CHUNK_FAILED); + } promise.completeExceptionally(error); - if(error instanceof Error) { + if (error instanceof Error) { throw (Error) error; } @@ -213,12 +219,20 @@ class RangeQuery implements AsyncIterable { fetchOutstanding = true; nextChunk = null; - fetchingChunk = tr.getRange_internal(begin, end, - rowsLimited ? rowsRemaining : 0, 0, streamingMode.code(), + nextFuture = new CompletableFuture<>(); + final long sTime = System.nanoTime(); + fetchingChunk = tr.getRange_internal(begin, end, rowsLimited ? rowsRemaining : 0, 0, streamingMode.code(), ++iteration, snapshot, reverse); - nextFuture = new CompletableFuture<>(); - fetchingChunk.whenComplete(new FetchComplete(fetchingChunk, nextFuture)); + BiConsumer cons = new FetchComplete(fetchingChunk,nextFuture); + if(eventKeeper!=null){ + eventKeeper.increment(Events.RANGE_QUERY_FETCHES); + cons = cons.andThen((r,t)->{ + eventKeeper.timeNanos(Events.RANGE_QUERY_FETCH_TIME_NANOS, System.nanoTime()-sTime); + }); + } + + fetchingChunk.whenComplete(cons); } @Override @@ -233,7 +247,7 @@ class RangeQuery implements AsyncIterable { // We have a chunk and are still working though it if(index < chunk.values.size()) { - return AsyncUtil.READY_TRUE; + return AsyncUtil.READY_TRUE; } // If we are at the end of the current chunk there is either: @@ -267,6 +281,16 @@ class RangeQuery implements AsyncIterable { prevKey = result.getKey(); index++; + if (eventKeeper != null) { + // We record the BYTES_FETCHED here, rather than at a lower level, + // because some parts of the construction of a RangeResult occur underneath + // the JNI boundary, and we don't want to pass the eventKeeper down there + // (note: account for the length fields as well when recording the bytes + // fetched) + eventKeeper.count(Events.BYTES_FETCHED, result.getKey().length + result.getValue().length + 8); + eventKeeper.increment(Events.RANGE_QUERY_RECORDS_FETCHED); + } + // If this is the first call to next() on a chunk there cannot // be another waiting, since we could not have issued a request assert(!(initialNext && nextChunk != null)); diff --git a/bindings/java/src/main/com/apple/foundationdb/tuple/FastByteComparisons.java b/bindings/java/src/main/com/apple/foundationdb/tuple/FastByteComparisons.java index 84a66906d9..7f271cc476 100644 --- a/bindings/java/src/main/com/apple/foundationdb/tuple/FastByteComparisons.java +++ b/bindings/java/src/main/com/apple/foundationdb/tuple/FastByteComparisons.java @@ -72,6 +72,13 @@ abstract class FastByteComparisons { T buffer2, int offset2, int length2); } + /** + * @return a byte[] comparator for use in sorting, collections, and so on internally + * to the Java code. + */ + public static Comparator comparator(){ + return LexicographicalComparerHolder.getBestComparer(); + } /** * Pure Java Comparer * @@ -291,8 +298,4 @@ abstract class FastByteComparisons { } } } - - public static Comparator comparator() { - return LexicographicalComparerHolder.BEST_COMPARER; - } } diff --git a/bindings/java/src/tests.cmake b/bindings/java/src/tests.cmake index c65eb5a870..5820537173 100644 --- a/bindings/java/src/tests.cmake +++ b/bindings/java/src/tests.cmake @@ -32,6 +32,7 @@ set(JAVA_JUNIT_TESTS src/junit/com/apple/foundationdb/tuple/TuplePackingTest.java src/junit/com/apple/foundationdb/tuple/TupleSerializationTest.java src/junit/com/apple/foundationdb/RangeQueryTest.java + src/junit/com/apple/foundationdb/EventKeeperTest.java ) # Resources that are used in unit testing, but are not explicitly test files (JUnit rules, utility diff --git a/cmake/InstallLayout.cmake b/cmake/InstallLayout.cmake index dd95fb9e95..f4297f0179 100644 --- a/cmake/InstallLayout.cmake +++ b/cmake/InstallLayout.cmake @@ -380,10 +380,15 @@ set(CPACK_RPM_CLIENTS-VERSIONED_PRE_UNINSTALL_SCRIPT_FILE if (CMAKE_SYSTEM_PROCESSOR MATCHES "x86_64") set(CPACK_DEBIAN_CLIENTS-DEB_FILE_NAME "${deb-clients-filename}_amd64.deb") set(CPACK_DEBIAN_SERVER-DEB_FILE_NAME "${deb-server-filename}_amd64.deb") + set(CPACK_DEBIAN_CLIENTS-VERSIONED_FILE_NAME "${deb-clients-filename}.versioned_amd64.deb") + set(CPACK_DEBIAN_SERVER-VERSIONED_FILE_NAME "${deb-server-filename}.versioned_amd64.deb") else() set(CPACK_DEBIAN_CLIENTS-DEB_FILE_NAME "${deb-clients-filename}_${CMAKE_SYSTEM_PROCESSOR}.deb") set(CPACK_DEBIAN_SERVER-DEB_FILE_NAME "${deb-server-filename}_${CMAKE_SYSTEM_PROCESSOR}.deb") + set(CPACK_DEBIAN_CLIENTS-VERSIONED_FILE_NAME "${deb-clients-filename}.versioned_${CMAKE_SYSTEM_PROCESSOR}.deb") + set(CPACK_DEBIAN_SERVER-VERSIONED_FILE_NAME "${deb-server-filename}.versioned_${CMAKE_SYSTEM_PROCESSOR}.deb") endif() + set(CPACK_DEB_COMPONENT_INSTALL ON) set(CPACK_DEBIAN_DEBUGINFO_PACKAGE ${GENERATE_DEBUG_PACKAGES}) set(CPACK_DEBIAN_PACKAGE_SECTION "database") diff --git a/documentation/sphinx/source/release-notes/release-notes-630.rst b/documentation/sphinx/source/release-notes/release-notes-630.rst index 02f42c6dd9..219f2d77fe 100644 --- a/documentation/sphinx/source/release-notes/release-notes-630.rst +++ b/documentation/sphinx/source/release-notes/release-notes-630.rst @@ -2,6 +2,10 @@ Release Notes ############# +6.3.12 +====== +* Change the default for --knob_tls_server_handshake_threads to 64. The previous was 1000. This avoids starting 1000 threads by default, but may adversely affect recovery time for large clusters using tls. Users with large tls clusters should consider explicitly setting this knob in their foundationdb.conf file. `(PR #4421) `_ + 6.3.11 ====== * Added a hint field in the trace event when all replicas of some data are lost. `(PR #4209) `_ diff --git a/fdbbackup/backup.actor.cpp b/fdbbackup/backup.actor.cpp index badd2dacf2..ba5c71486d 100644 --- a/fdbbackup/backup.actor.cpp +++ b/fdbbackup/backup.actor.cpp @@ -2553,8 +2553,8 @@ ACTOR Future expireBackupData(const char* name, throw; if (e.code() == error_code_backup_cannot_expire) fprintf(stderr, - "ERROR: Requested expiration would be unsafe. Backup would not meet minimum " - "restorability. Use --force to delete data anyway.\n"); + "ERROR: Requested expiration would be unsafe. Backup would not meet minimum restorability. Use " + "--force to delete data anyway.\n"); else fprintf(stderr, "ERROR: %s\n", e.what()); throw; @@ -4271,4 +4271,4 @@ int main(int argc, char* argv[]) { } flushAndExit(status); -} +} \ No newline at end of file diff --git a/fdbclient/BackupContainer.actor.cpp b/fdbclient/BackupContainer.actor.cpp index 1388941dd6..a71ab0c6ff 100644 --- a/fdbclient/BackupContainer.actor.cpp +++ b/fdbclient/BackupContainer.actor.cpp @@ -58,7 +58,6 @@ ACTOR Future appendStringRefWithLen(Reference file, Standalon wait(file->append(s.begin(), s.size())); return Void(); } - } // namespace IBackupFile_impl Future IBackupFile::appendStringRefWithLen(Standalone s) { diff --git a/fdbrpc/sim2.actor.cpp b/fdbrpc/sim2.actor.cpp index 03e7081048..7e3f143185 100644 --- a/fdbrpc/sim2.actor.cpp +++ b/fdbrpc/sim2.actor.cpp @@ -1056,6 +1056,7 @@ public: bool isAddressOnThisHost(NetworkAddress const& addr) const override { return addr.ip == getCurrentProcess()->address.ip; } + virtual bool isAddressOnThisHost(NetworkAddress const& addr) { return addr.ip == getCurrentProcess()->address.ip; } ACTOR static Future deleteFileImpl(Sim2* self, std::string filename, bool mustBeDurable) { // This is a _rudimentary_ simulation of the untrustworthiness of non-durable deletes and the possibility of diff --git a/fdbserver/CommitProxyServer.actor.cpp b/fdbserver/CommitProxyServer.actor.cpp index 551bfde457..afd6475f37 100644 --- a/fdbserver/CommitProxyServer.actor.cpp +++ b/fdbserver/CommitProxyServer.actor.cpp @@ -384,7 +384,7 @@ ACTOR Future addBackupMutations(ProxyCommitData* self, // describe(tags)).detail("BackupMutation", backupMutation.toString()) // .detail("BackupMutationSize", val.size()).detail("Version", commitVersion).detail("DestPath", // logRangeMutation.first) .detail("PartIndex", part).detail("PartIndexEndian", - //bigEndian32(part)).detail("PartData", backupMutation.param1); + // bigEndian32(part)).detail("PartData", backupMutation.param1); // } } } diff --git a/fdbserver/Resolver.actor.cpp b/fdbserver/Resolver.actor.cpp index d97aac6e36..a2d769b1fc 100644 --- a/fdbserver/Resolver.actor.cpp +++ b/fdbserver/Resolver.actor.cpp @@ -30,7 +30,6 @@ #include "fdbserver/Orderer.actor.h" #include "fdbserver/StorageMetrics.h" #include "fdbclient/SystemData.h" - #include "flow/actorcompiler.h" // This must be the last #include. namespace { diff --git a/fdbserver/SimulatedCluster.actor.cpp b/fdbserver/SimulatedCluster.actor.cpp index 55a7186bc0..480392984b 100644 --- a/fdbserver/SimulatedCluster.actor.cpp +++ b/fdbserver/SimulatedCluster.actor.cpp @@ -1432,7 +1432,6 @@ void setupSimulatedSystem(vector>* systemActors, std::vector processClassesSubSet = { ProcessClass::UnsetClass, ProcessClass::ResolutionClass, ProcessClass::MasterClass }; - for (int dc = 0; dc < dataCenters; dc++) { // FIXME: test unset dcID Optional> dcUID = StringRef(format("%d", dc)); diff --git a/flow/Knobs.cpp b/flow/Knobs.cpp index 4aec8afe5a..a173ba43bf 100644 --- a/flow/Knobs.cpp +++ b/flow/Knobs.cpp @@ -96,7 +96,7 @@ void FlowKnobs::initialize(bool randomize, bool isSimulated) { init( TLS_SERVER_CONNECTION_THROTTLE_ATTEMPTS, 1 ); init( TLS_CLIENT_CONNECTION_THROTTLE_ATTEMPTS, 1 ); init( TLS_CLIENT_HANDSHAKE_THREADS, 0 ); - init( TLS_SERVER_HANDSHAKE_THREADS, 1000 ); + init( TLS_SERVER_HANDSHAKE_THREADS, 64 ); init( TLS_HANDSHAKE_THREAD_STACKSIZE, 64 * 1024 ); init( TLS_MALLOC_ARENA_MAX, 6 ); init( TLS_HANDSHAKE_LIMIT, 1000 ); diff --git a/flow/Net2.actor.cpp b/flow/Net2.actor.cpp index 3a6a4a24dd..3a1e52dc4a 100644 --- a/flow/Net2.actor.cpp +++ b/flow/Net2.actor.cpp @@ -1742,8 +1742,8 @@ THREAD_HANDLE Net2::startThread(THREAD_FUNC_RETURN (*func)(void*), void* arg) { Future> Net2::connect(NetworkAddress toAddr, const std::string& host) { #ifndef TLS_DISABLED - initTLS(ETLSInitState::CONNECT); if (toAddr.isTLS()) { + initTLS(ETLSInitState::CONNECT); return SSLConnection::connect(&this->reactor.ios, this->sslContextVar.get(), toAddr); } #endif @@ -1839,8 +1839,8 @@ bool Net2::isAddressOnThisHost(NetworkAddress const& addr) const { Reference Net2::listen(NetworkAddress localAddr) { try { #ifndef TLS_DISABLED - initTLS(ETLSInitState::LISTEN); if (localAddr.isTLS()) { + initTLS(ETLSInitState::LISTEN); return Reference(new SSLListener(reactor.ios, &this->sslContextVar, localAddr)); } #endif diff --git a/flow/folly_memcpy.h b/flow/folly_memcpy.h index 6ef644dc4a..7f1b8e443a 100644 --- a/flow/folly_memcpy.h +++ b/flow/folly_memcpy.h @@ -30,4 +30,4 @@ void* folly_memcpy(void* dst, const void* src, uint32_t length); #endif // linux or bsd and avx -#endif \ No newline at end of file +#endif diff --git a/packaging/multiversion/clients/postinst b/packaging/multiversion/clients/postinst index f982bda675..49ba21db61 100644 --- a/packaging/multiversion/clients/postinst +++ b/packaging/multiversion/clients/postinst @@ -13,7 +13,7 @@ then fi -alternatives --install /usr/bin/fdbcli fdbclients /usr/lib/foundationdb-@PROJECT_VERSION@/bin/fdbcli @ALTERNATIVES_PRIORITY@ \ +update-alternatives --install /usr/bin/fdbcli fdbclients /usr/lib/foundationdb-@PROJECT_VERSION@/bin/fdbcli @ALTERNATIVES_PRIORITY@ \ --slave /usr/bin/fdbbackup fdbbackup /usr/lib/foundationdb-@PROJECT_VERSION@/bin/fdbbackup \ --slave /usr/bin/fdbrestore fdbrestore /usr/lib/foundationdb-@PROJECT_VERSION@/bin/fdbbackup \ --slave /usr/bin/dr_agent dr_agent /usr/lib/foundationdb-@PROJECT_VERSION@/bin/fdbbackup \ diff --git a/packaging/multiversion/clients/prerm b/packaging/multiversion/clients/prerm index 252f46ac63..8386af0219 100644 --- a/packaging/multiversion/clients/prerm +++ b/packaging/multiversion/clients/prerm @@ -1,3 +1,3 @@ #!/usr/bin/env bash -alternatives --remove fdbclients /usr/lib/foundationdb-@PROJECT_VERSION@/bin/fdbcli +update-alternatives --remove fdbclients /usr/lib/foundationdb-@PROJECT_VERSION@/bin/fdbcli diff --git a/packaging/multiversion/server/postinst b/packaging/multiversion/server/postinst index d0d8867b43..1951559fd9 100644 --- a/packaging/multiversion/server/postinst +++ b/packaging/multiversion/server/postinst @@ -1,4 +1,4 @@ #!/usr/bin/env bash -alternatives --install /usr/sbin/fdbserver fdbserver /usr/lib/foundationdb-@PROJECT_VERSION@/sbin/fdbserver @ALTERNATIVES_PRIORITY@ \ +update-alternatives --install /usr/sbin/fdbserver fdbserver /usr/lib/foundationdb-@PROJECT_VERSION@/sbin/fdbserver @ALTERNATIVES_PRIORITY@ \ --slave /usr/sbin/fdbmonitor fdbmonitor /usr/lib/foundationdb-@PROJECT_VERSION@/sbin/fdbmonitor diff --git a/packaging/multiversion/server/prerm b/packaging/multiversion/server/prerm index c3dab2d7e7..b981addedd 100644 --- a/packaging/multiversion/server/prerm +++ b/packaging/multiversion/server/prerm @@ -1,3 +1,3 @@ #!/usr/bin/env bash -alternatives --remove fdbserver /usr/lib/foundationdb-@PROJECT_VERSION@/sbin/fdbserver +update-alternatives --remove fdbserver /usr/lib/foundationdb-@PROJECT_VERSION@/sbin/fdbserver