wbtiger
/
foundationdb
mirror of https://github.com/apple/foundationdb.git
0
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity
foundationdb/fdbserver/resolver/ConflictSet.cpp

1515 lines
46 KiB
C++
Raw Blame History

/*
* ConflictSet.cpp
*
* This source file is part of the FoundationDB open source project
*
* Copyright 2013-2026 Apple Inc. and the FoundationDB project authors
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
#include <stdint.h>
#include <memory.h>
#include <stdio.h>
#include <algorithm>
#include <numeric>
#include <string>
#include <vector>
#include "flow/Platform.h"
#include "fdbrpc/fdbrpc.h"
#include "fdbrpc/PerfMetric.h"
#include "fdbclient/FDBTypes.h"
#include "fdbclient/KeyRangeMap.h"
#include "fdbclient/SystemData.h"
#include "ConflictSet.h"
#include "flow/UnitTest.h"
static std::vector<PerfDoubleCounter*> skc;
static thread_local uint32_t g_seed = 0;
static inline int skfastrand() {
g_seed = g_seed * 1664525L + 1013904223L;
return g_seed;
}
PerfDoubleCounter g_buildTest("Build", skc), g_add("Add", skc), g_detectConflicts("Detect", skc), g_sort("D.Sort", skc),
g_combine("D.Combine", skc), g_checkRead("D.CheckRead", skc), g_checkBatch("D.CheckIntraBatch", skc),
g_merge("D.MergeWrite", skc), g_removeBefore("D.RemoveBefore", skc);
static force_inline int compare(const StringRef& a, const StringRef& b) {
const size_t aSize = a.size();
const size_t bSize = b.size();
const size_t minSize = std::min(aSize, bSize);
int c = memcmp(a.begin(), b.begin(), minSize);
if (c)
return (c > 0) - (c < 0); // normalize to +1/-1
return (aSize > bSize) - (aSize < bSize);
}
struct ReadConflictRange {
StringRef begin, end;
Version version;
int transaction;
int indexInTx;
VectorRef<int>* conflictingKeyRange;
Arena* cKRArena;
ReadConflictRange(StringRef begin,
StringRef end,
Version version,
int transaction,
int indexInTx,
VectorRef<int>* cKR = nullptr,
Arena* cKRArena = nullptr)
: begin(begin), end(end), version(version), transaction(transaction), indexInTx(indexInTx),
conflictingKeyRange(cKR), cKRArena(cKRArena) {}
bool operator<(const ReadConflictRange& rhs) const { return compare(begin, rhs.begin) < 0; }
};
struct KeyInfo {
StringRef key;
int* pIndex;
bool begin;
bool write;
int transaction;
KeyInfo() = default;
KeyInfo(StringRef key, bool begin, bool write, int transaction, int* pIndex)
: key(key), pIndex(pIndex), begin(begin), write(write), transaction(transaction) {}
};
force_inline int extra_ordering(const KeyInfo& ki) {
return ki.begin * 2 + (ki.write ^ ki.begin);
}
// returns true if done with string
force_inline bool getCharacter(const KeyInfo& ki, int character, int& outputCharacter) {
// normal case
if (character < ki.key.size()) {
outputCharacter = 5 + ki.key.begin()[character];
return false;
}
// termination
if (character == ki.key.size()) {
outputCharacter = 0;
return false;
}
if (character == ki.key.size() + 1) {
// end/begin+read/write relative sorting
outputCharacter = extra_ordering(ki);
return false;
}
outputCharacter = 0;
return true;
}
bool operator<(const KeyInfo& lhs, const KeyInfo& rhs) {
int i = std::min(lhs.key.size(), rhs.key.size());
int c = memcmp(lhs.key.begin(), rhs.key.begin(), i);
if (c != 0)
return c < 0;
// Always sort shorter keys before longer keys.
if (lhs.key.size() < rhs.key.size()) {
return true;
}
if (lhs.key.size() > rhs.key.size()) {
return false;
}
// When the keys are the same length, use the extra ordering constraint.
return extra_ordering(lhs) < extra_ordering(rhs);
}
bool operator==(const KeyInfo& lhs, const KeyInfo& rhs) {
return !(lhs < rhs || rhs < lhs);
}
void swapSort(std::vector<KeyInfo>& points, int a, int b) {
if (points[b] < points[a]) {
KeyInfo temp;
temp = points[a];
points[a] = points[b];
points[b] = temp;
}
}
void smallSort(std::vector<KeyInfo>& points, int start, int N) {
for (int i = 1; i < N; i++)
for (int j = i; j > 0; j -= 2)
swapSort(points, start + j - 1, start + j);
for (int i = N - 2; i > 0; i--)
for (int j = i; j > 0; j -= 2)
swapSort(points, start + j - 1, start + j);
}
struct SortTask {
int begin;
int size;
int character;
SortTask(int begin, int size, int character) : begin(begin), size(size), character(character) {}
};
void sortPoints(std::vector<KeyInfo>& points) {
std::vector<SortTask> tasks;
std::vector<KeyInfo> newPoints;
std::vector<int> counts;
tasks.reserve(points.size());
newPoints.reserve(points.size());
counts.resize(261); // 256+5 = character+sentinal
tasks.emplace_back(0, points.size(), 0);
while (!tasks.empty()) {
SortTask st = tasks.back();
tasks.pop_back();
if (st.size < 10) {
// smallSort(points, st.begin, st.size);
std::sort(points.begin() + st.begin, points.begin() + st.begin + st.size);
continue;
}
newPoints.resize(st.size);
std::fill(counts.begin(), counts.end(), 0);
// get counts
int c;
bool allDone = true;
for (int i = st.begin; i < st.begin + st.size; i++) {
allDone &= getCharacter(points[i], st.character, c);
counts[c]++;
}
if (allDone)
continue;
// calculate offsets from counts and build next level of tasks
int total = 0;
for (int i = 0; i < counts.size(); i++) {
int temp = counts[i];
if (temp > 1)
tasks.emplace_back(st.begin + total, temp, st.character + 1);
counts[i] = total;
total += temp;
}
// put in their places
for (int i = st.begin; i < st.begin + st.size; i++) {
getCharacter(points[i], st.character, c);
newPoints[counts[c]++] = points[i];
}
// copy back into original points array
for (int i = 0; i < st.size; i++)
points[st.begin + i] = newPoints[i];
}
}
class SkipList : NonCopyable {
private:
static constexpr int MaxLevels = 26;
int randomLevel() const {
uint32_t i = uint32_t(skfastrand()) >> (32 - (MaxLevels - 1));
int level = 0;
while (i & 1) {
i >>= 1;
level++;
}
ASSERT(level < MaxLevels);
return level;
}
// Represent a node in the SkipList. The node has multiple (i.e., level) pointers to
// other nodes, and keeps a record of the max versions for each level.
struct Node {
int level() const { return nPointers - 1; }
uint8_t* value() { return end() + nPointers * (sizeof(Node*) + sizeof(Version)); }
int length() const { return valueLength; }
// Returns the next node pointer at the given level.
Node* getNext(int level) { return *((Node**)end() + level); }
// Sets the next node pointer at the given level.
void setNext(int level, Node* n) { *((Node**)end() + level) = n; }
// Returns the max version at the given level.
Version getMaxVersion(int i) const { return ((Version*)(end() + nPointers * sizeof(Node*)))[i]; }
// Sets the max version at the given level.
void setMaxVersion(int i, Version v) { ((Version*)(end() + nPointers * sizeof(Node*)))[i] = v; }
// Return a node with initialized value but uninitialized pointers
// Memory layout: *this, (level+1) Node*, (level+1) Version, value
static Node* create(const StringRef& value, int level) {
int nodeSize = sizeof(Node) + value.size() + (level + 1) * (sizeof(Node*) + sizeof(Version));
Node* n;
if (nodeSize <= 64) {
n = (Node*)FastAllocator<64>::allocate();
INSTRUMENT_ALLOCATE("SkipListNode64");
} else if (nodeSize <= 128) {
n = (Node*)FastAllocator<128>::allocate();
INSTRUMENT_ALLOCATE("SkipListNode128");
} else {
n = (Node*)new char[nodeSize];
INSTRUMENT_ALLOCATE("SkipListNodeLarge");
}
n->nPointers = level + 1;
n->valueLength = value.size();
if (!value.empty()) {
memcpy(n->value(), value.begin(), value.size());
}
return n;
}
// pre: level>0, all lower level nodes between this and getNext(level) have correct maxversions
void calcVersionForLevel(int level) {
Node* end = getNext(level);
Version v = getMaxVersion(level - 1);
for (Node* x = getNext(level - 1); x != end; x = x->getNext(level - 1))
v = std::max(v, x->getMaxVersion(level - 1));
setMaxVersion(level, v);
}
void destroy() {
int nodeSize = getNodeSize();
if (nodeSize <= 64) {
FastAllocator<64>::release(this);
INSTRUMENT_RELEASE("SkipListNode64");
} else if (nodeSize <= 128) {
FastAllocator<128>::release(this);
INSTRUMENT_RELEASE("SkipListNode128");
} else {
delete[] (char*)this;
INSTRUMENT_RELEASE("SkipListNodeLarge");
}
}
private:
int getNodeSize() const { return sizeof(Node) + valueLength + nPointers * (sizeof(Node*) + sizeof(Version)); }
// Returns the first Node* pointer
uint8_t* end() { return (uint8_t*)(this + 1); }
uint8_t const* end() const { return (uint8_t const*)(this + 1); }
int nPointers, valueLength;
};
static force_inline bool less(const uint8_t* a, int aLen, const uint8_t* b, int bLen) {
int c = memcmp(a, b, std::min(aLen, bLen));
if (c < 0)
return true;
if (c > 0)
return false;
return aLen < bLen;
}
Node* header;
void destroy() {
Node *next, *x;
for (x = header; x; x = next) {
next = x->getNext(0);
x->destroy();
}
}
public:
// Points the location (i.e., Node*) that value would appear in the SkipList.
// If the "value" is in the list, then finger[0] points to that exact node;
// otherwise, the finger points to Nodes that the value should be inserted before.
// Note the SkipList organizes all nodes at level 0, higher levels contain jump pointers.
struct Finger {
Node* finger[MaxLevels]; // valid for levels >= level
int level = MaxLevels;
Node* x = nullptr;
Node* alreadyChecked = nullptr;
StringRef value;
Finger() = default;
Finger(Node* header, const StringRef& ptr) : x(header), value(ptr) {}
void init(const StringRef& value, Node* header) {
this->value = value;
x = header;
alreadyChecked = nullptr;
level = MaxLevels;
}
// pre: !finished()
force_inline void prefetch() {
Node* next = x->getNext(level - 1);
if (next) {
_mm_prefetch((const char*)next, _MM_HINT_T0);
_mm_prefetch((const char*)next + 64, _MM_HINT_T0);
}
}
// pre: !finished()
// Advances the pointer at the current level to a Node that's >= finger's value
// if possible; or move to the next level (i.e., level--).
// Returns true if we have advanced to the next level
force_inline bool advance() {
Node* next = x->getNext(level - 1);
if (next == alreadyChecked || !less(next->value(), next->length(), value.begin(), value.size())) {
alreadyChecked = next;
level--;
finger[level] = x;
return true;
} else {
x = next;
return false;
}
}
// pre: !finished()
force_inline void nextLevel() {
while (!advance())
;
}
force_inline bool finished() const { return level == 0; }
// Returns if the finger value is found in the SkipList.
force_inline Node* found() const {
// valid after finished returns true
Node* n = finger[0]->getNext(0); // or alreadyChecked, but that is more easily invalidated
if (n && n->length() == value.size() && !memcmp(n->value(), value.begin(), value.size()))
return n;
else
return nullptr;
}
StringRef getValue() const {
Node* n = finger[0]->getNext(0);
return n ? StringRef(n->value(), n->length()) : StringRef();
}
};
// Returns the total number of nodes in the list.
int count() const {
int count = 0;
Node* x = header->getNext(0);
while (x) {
x = x->getNext(0);
count++;
}
return count;
}
explicit SkipList(Version version = 0) {
header = Node::create(StringRef(), MaxLevels - 1);
for (int l = 0; l < MaxLevels; l++) {
header->setNext(l, nullptr);
header->setMaxVersion(l, version);
}
}
~SkipList() { destroy(); }
explicit(false) SkipList(SkipList&& other) noexcept : header(other.header) { other.header = nullptr; }
void operator=(SkipList&& other) noexcept {
destroy();
header = other.header;
other.header = nullptr;
}
void swap(SkipList& other) { std::swap(header, other.header); }
void addConflictRanges(const Finger* fingers, int rangeCount, Version version) {
for (int r = rangeCount - 1; r >= 0; r--) {
const size_t fingerIndex = static_cast<size_t>(r) * 2;
const Finger& startF = fingers[fingerIndex];
const Finger& endF = fingers[fingerIndex + 1];
if (endF.found() == nullptr)
insert(endF, endF.finger[0]->getMaxVersion(0));
remove(startF, endF);
insert(startF, version);
}
}
void detectConflicts(ReadConflictRange* ranges, int count, bool* transactionConflictStatus) {
const int M = 16;
int nextJob[M];
CheckMax inProgress[M];
if (!count)
return;
int started = std::min(M, count);
for (int i = 0; i < started; i++) {
inProgress[i].init(ranges[i],
header,
transactionConflictStatus,
ranges[i].indexInTx,
ranges[i].conflictingKeyRange,
ranges[i].cKRArena);
nextJob[i] = i + 1;
}
nextJob[started - 1] = 0;
int prevJob = started - 1;
int job = 0;
// vtune: 340 parts
while (true) {
if (inProgress[job].advance()) {
if (started == count) {
if (prevJob == job)
break;
nextJob[prevJob] = nextJob[job];
job = prevJob;
} else {
int temp = started++;
inProgress[job].init(ranges[temp],
header,
transactionConflictStatus,
ranges[temp].indexInTx,
ranges[temp].conflictingKeyRange,
ranges[temp].cKRArena);
}
}
prevJob = job;
job = nextJob[job];
}
}
void find(const StringRef* values, Finger* results, int* temp, int count) {
// Relying on the ordering of values, descend until the values aren't all in the
// same part of the tree
// vtune: 11 parts
results[0].init(values[0], header);
const StringRef& endValue = values[count - 1];
while (results[0].level > 1) {
results[0].nextLevel();
Node* ac = results[0].alreadyChecked;
if (ac && less(ac->value(), ac->length(), endValue.begin(), endValue.size()))
break;
}
// Init all the other fingers to start descending where we stopped
// the first one
// SOMEDAY: this loop showed up on vtune, could be faster?
// vtune: 8 parts
int startLevel = results[0].level + 1;
Node* x = startLevel < MaxLevels ? results[0].finger[startLevel] : header;
for (int i = 1; i < count; i++) {
results[i].level = startLevel;
results[i].x = x;
results[i].alreadyChecked = nullptr;
results[i].value = values[i];
for (int j = startLevel; j < MaxLevels; j++)
results[i].finger[j] = results[0].finger[j];
}
int* nextJob = temp;
for (int i = 0; i < count - 1; i++)
nextJob[i] = i + 1;
nextJob[count - 1] = 0;
int prevJob = count - 1;
int job = 0;
// vtune: 225 parts
while (true) {
Finger* f = &results[job];
f->advance();
if (f->finished()) {
if (prevJob == job)
break;
nextJob[prevJob] = nextJob[job];
} else {
f->prefetch();
prevJob = job;
}
job = nextJob[job];
}
}
int removeBefore(Version v, Finger& f, int nodeCount) {
// f.x, f.alreadyChecked?
int removedCount = 0;
bool wasAbove = true;
while (nodeCount--) {
Node* x = f.finger[0]->getNext(0);
if (!x)
break;
// double prefetch gives +25% speed (single threaded)
Node* next = x->getNext(0);
_mm_prefetch((const char*)next, _MM_HINT_T0);
next = x->getNext(1);
_mm_prefetch((const char*)next, _MM_HINT_T0);
bool isAbove = x->getMaxVersion(0) >= v;
if (isAbove || wasAbove) { // f.nextItem
for (int l = 0; l <= x->level(); l++)
f.finger[l] = x;
} else { // f.eraseItem
removedCount++;
for (int l = 0; l <= x->level(); l++)
f.finger[l]->setNext(l, x->getNext(l));
for (int i = 1; i <= x->level(); i++)
f.finger[i]->setMaxVersion(i, std::max(f.finger[i]->getMaxVersion(i), x->getMaxVersion(i)));
x->destroy();
}
wasAbove = isAbove;
}
return removedCount;
}
private:
void remove(const Finger& start, const Finger& end) {
if (start.finger[0] == end.finger[0])
return;
Node* x = start.finger[0]->getNext(0);
// vtune says: this loop is the expensive parts (6 parts)
for (int i = 0; i < MaxLevels; i++)
if (start.finger[i] != end.finger[i])
start.finger[i]->setNext(i, end.finger[i]->getNext(i));
while (true) {
Node* next = x->getNext(0);
x->destroy();
if (x == end.finger[0])
break;
x = next;
}
}
void insert(const Finger& f, Version version) {
int level = randomLevel();
// std::cout << std::string((const char*)value,length) << " level: " << level << std::endl;
Node* x = Node::create(f.value, level);
x->setMaxVersion(0, version);
for (int i = 0; i <= level; i++) {
x->setNext(i, f.finger[i]->getNext(i));
f.finger[i]->setNext(i, x);
}
// vtune says: this loop is the costly part of this function
for (int i = 1; i <= level; i++) {
f.finger[i]->calcVersionForLevel(i);
x->calcVersionForLevel(i);
}
for (int i = level + 1; i < MaxLevels; i++) {
Version v = f.finger[i]->getMaxVersion(i);
if (v >= version)
break;
f.finger[i]->setMaxVersion(i, version);
}
}
void insert(const StringRef& value, Version version) {
Finger f(header, value);
while (!f.finished())
f.nextLevel();
// SOMEDAY: equality?
insert(f, version);
}
struct CheckMax {
Finger start, end;
Version version;
bool* result;
int state;
int indexInTx;
VectorRef<int>* conflictingKeyRange; // nullptr if report_conflicting_keys is not enabled.
Arena* cKRArena; // nullptr if report_conflicting_keys is not enabled.
void init(const ReadConflictRange& r,
Node* header,
bool* tCS,
int indexInTx,
VectorRef<int>* cKR,
Arena* cKRArena) {
this->start.init(r.begin, header);
this->end.init(r.end, header);
this->version = r.version;
this->indexInTx = indexInTx;
this->cKRArena = cKRArena;
result = &tCS[r.transaction];
conflictingKeyRange = cKR;
this->state = 0;
}
bool noConflict() const { return true; }
bool conflict() {
*result = true;
if (conflictingKeyRange != nullptr)
conflictingKeyRange->push_back(*cKRArena, indexInTx);
return true;
}
// Return true if finished
force_inline bool advance() {
switch (state) {
case 0:
// find where start and end fingers diverge
while (true) {
if (!start.advance()) {
start.prefetch();
return false;
}
end.x = start.x;
while (!end.advance())
;
int l = start.level;
if (start.finger[l] != end.finger[l])
break;
// accept if the range spans the check range, but does not have a greater version
if (start.finger[l]->getMaxVersion(l) <= version)
return noConflict();
if (l == 0)
return conflict();
}
state = 1;
case 1: {
// check the end side of the pyramid
Node* e = end.finger[end.level];
while (e->getMaxVersion(end.level) > version) {
if (end.finished())
return conflict();
end.nextLevel();
Node* f = end.finger[end.level];
while (e != f) {
if (e->getMaxVersion(end.level) > version)
return conflict();
e = e->getNext(end.level);
}
}
// check the start side of the pyramid
Node* s = end.finger[start.level];
while (true) {
Node* nextS = start.finger[start.level]->getNext(start.level);
Node* p = nextS;
while (p != s) {
if (p->getMaxVersion(start.level) > version)
return conflict();
p = p->getNext(start.level);
}
if (start.finger[start.level]->getMaxVersion(start.level) <= version)
return noConflict();
s = nextS;
if (start.finished()) {
if (nextS->length() == start.value.size() &&
!memcmp(nextS->value(), start.value.begin(), start.value.size()))
return noConflict();
else
return conflict();
}
start.nextLevel();
}
}
default:
__assume(false);
}
}
};
// Splits the SkipLists so that those after finger is moved to "right".
void split(const Finger& f, SkipList& right) {
ASSERT(!right.header->getNext(0)); // right must be empty
right.header->setMaxVersion(0, f.finger[0]->getMaxVersion(0));
for (int l = 0; l < MaxLevels; l++) {
right.header->setNext(l, f.finger[l]->getNext(l));
f.finger[l]->setNext(l, nullptr);
}
}
// Sets end's finger to the last nodes at all levels.
void getEnd(Finger& end) {
Node* node = header;
for (int l = MaxLevels - 1; l >= 0; l--) {
Node* next;
while ((next = node->getNext(l)) != nullptr)
node = next;
end.finger[l] = node;
}
end.level = 0;
}
};
struct ConflictSet {
ConflictSet() : removalKey(makeString(0)), oldestVersion(0) {}
~ConflictSet() = default;
SkipList versionHistory;
Key removalKey;
Version oldestVersion;
};
ConflictSet* newConflictSet() {
return new ConflictSet;
}
void clearConflictSet(ConflictSet* cs, Version v) {
SkipList(v).swap(cs->versionHistory);
}
void destroyConflictSet(ConflictSet* cs) {
delete cs;
}
ConflictBatch::ConflictBatch(ConflictSet* cs,
std::map<int, VectorRef<int>>* conflictingKeyRangeMap,
Arena* resolveBatchReplyArena)
: cs(cs), transactionCount(0), conflictingKeyRangeMap(conflictingKeyRangeMap),
resolveBatchReplyArena(resolveBatchReplyArena) {}
ConflictBatch::~ConflictBatch() = default;
struct TransactionInfo {
VectorRef<std::pair<int, int>> readRanges;
VectorRef<std::pair<int, int>> writeRanges;
bool tooOld;
bool reportConflictingKeys;
};
bool ConflictBatch::ignoreTooOld() const {
return bugs && deterministicRandom()->random01() < bugs->ignoreTooOldProbability;
}
bool ConflictBatch::ignoreReadSet() const {
return bugs && deterministicRandom()->random01() < bugs->ignoreReadSetProbability;
}
bool ConflictBatch::ignoreWriteSet() const {
return bugs && deterministicRandom()->random01() < bugs->ignoreWriteSetProbability;
}
void ConflictBatch::addTransaction(const CommitTransactionRef& tr, Version newOldestVersion) {
const int t = transactionCount++;
Arena& arena = transactionInfo.arena();
auto* info = new (arena) TransactionInfo;
info->reportConflictingKeys = tr.report_conflicting_keys;
bool tooOld = tr.read_snapshot < newOldestVersion && !tr.read_conflict_ranges.empty();
if (tooOld && ignoreTooOld()) {
bugs->hit();
tooOld = false;
}
if (tooOld) {
info->tooOld = true;
} else {
info->tooOld = false;
if (!ignoreReadSet()) {
info->readRanges.resize(arena, tr.read_conflict_ranges.size());
} else {
bugs->hit();
}
if (!ignoreWriteSet()) {
info->writeRanges.resize(arena, tr.write_conflict_ranges.size());
} else {
bugs->hit();
}
for (int r = 0; r < info->readRanges.size(); r++) {
const KeyRangeRef& range = tr.read_conflict_ranges[r];
points.emplace_back(range.begin, true, false, t, &info->readRanges[r].first);
points.emplace_back(range.end, false, false, t, &info->readRanges[r].second);
combinedReadConflictRanges.emplace_back(range.begin,
range.end,
tr.read_snapshot,
t,
r,
tr.report_conflicting_keys ? &(*conflictingKeyRangeMap)[t]
: nullptr,
tr.report_conflicting_keys ? resolveBatchReplyArena : nullptr);
}
for (int r = 0; r < info->writeRanges.size(); r++) {
const KeyRangeRef& range = tr.write_conflict_ranges[r];
points.emplace_back(range.begin, true, true, t, &info->writeRanges[r].first);
points.emplace_back(range.end, false, true, t, &info->writeRanges[r].second);
}
}
transactionInfo.push_back(arena, info);
}
class MiniConflictSet : NonCopyable {
std::vector<uint64_t> words;
public:
explicit MiniConflictSet(size_t size) {
ASSERT(size <= std::numeric_limits<size_t>::max() - 63); // Prevent overflow in (size + 63)
words.resize((size + 63) / 64, 0);
}
void set(int begin, int end) {
if (end <= begin)
return;
const size_t wordBegin = static_cast<size_t>(begin) / 64;
const size_t wordEnd = static_cast<size_t>(end - 1) / 64; // Last word containing bits (inclusive)
if (wordBegin == wordEnd) {
// Single word case
const size_t bitStart = static_cast<size_t>(begin) % 64;
const size_t numBits = static_cast<size_t>(end - begin);
ASSERT(numBits <= 64); // Single word should never span > 64 bits
uint64_t mask;
if (numBits == 64) {
mask = ~0ULL;
} else {
mask = ((1ULL << numBits) - 1) << bitStart;
}
words[wordBegin] |= mask;
} else {
// Multi-word case: wordBegin (partial) + middle words (full) + wordEnd (partial)
words[wordBegin] |= ~0ULL << (static_cast<size_t>(begin) % 64);
for (size_t w = wordBegin + 1; w < wordEnd; w++) { // Fill middle words completely
words[w] = ~0ULL;
}
words[wordEnd] |= ~0ULL >> (63 - (static_cast<size_t>(end - 1) % 64));
}
}
bool any(int begin, int end) const {
if (end <= begin)
return false;
const size_t wordBegin = static_cast<size_t>(begin) / 64;
const size_t wordEnd = static_cast<size_t>(end - 1) / 64; // Last word containing bits (inclusive)
for (size_t w = wordBegin; w <= wordEnd; w++) { // Check all words including wordEnd
uint64_t mask = ~0ULL;
if (w == wordBegin) {
mask &= (~0ULL << (static_cast<size_t>(begin) % 64));
}
if (w == wordEnd) {
mask &= (~0ULL >> (63 - (static_cast<size_t>(end - 1) % 64)));
}
if (words[w] & mask)
return true;
}
return false;
}
void clear() { std::fill(words.begin(), words.end(), 0); }
};
void ConflictBatch::checkIntraBatchConflicts() {
int index = 0;
for (int p = 0; p < points.size(); p++)
*points[p].pIndex = index++;
MiniConflictSet mcs(index);
for (int t = 0; t < transactionInfo.size(); t++) {
const TransactionInfo& tr = *transactionInfo[t];
if (transactionConflictStatus[t])
continue;
bool conflict = tr.tooOld;
for (int i = 0; i < tr.readRanges.size(); i++) {
if (mcs.any(tr.readRanges[i].first, tr.readRanges[i].second)) {
if (tr.reportConflictingKeys) {
(*conflictingKeyRangeMap)[t].push_back(*resolveBatchReplyArena, i);
}
conflict = true;
break;
}
}
transactionConflictStatus[t] = conflict;
if (!conflict)
for (int i = 0; i < tr.writeRanges.size(); i++)
mcs.set(tr.writeRanges[i].first, tr.writeRanges[i].second);
}
}
void ConflictBatch::GetTooOldTransactions(std::vector<int>& tooOldTransactions) {
for (int i = 0; i < transactionInfo.size(); i++) {
if (transactionInfo[i]->tooOld) {
tooOldTransactions.push_back(i);
}
}
}
void ConflictBatch::detectConflicts(Version now,
Version newOldestVersion,
std::vector<int>& nonConflicting,
std::vector<int>* tooOldTransactions) {
double t = timer();
sortPoints(points);
g_sort += timer() - t;
transactionConflictStatus = new bool[transactionCount];
memset(transactionConflictStatus, 0, transactionCount * sizeof(bool));
t = timer();
checkReadConflictRanges();
g_checkRead += timer() - t;
t = timer();
checkIntraBatchConflicts();
g_checkBatch += timer() - t;
t = timer();
combineWriteConflictRanges();
g_combine += timer() - t;
t = timer();
mergeWriteConflictRanges(now);
g_merge += timer() - t;
for (int i = 0; i < transactionCount; i++) {
if (tooOldTransactions && transactionInfo[i]->tooOld) {
tooOldTransactions->push_back(i);
} else if (!transactionConflictStatus[i]) {
nonConflicting.push_back(i);
}
}
delete[] transactionConflictStatus;
t = timer();
if (newOldestVersion > cs->oldestVersion) {
cs->oldestVersion = newOldestVersion;
SkipList::Finger finger;
int temp;
cs->versionHistory.find(&cs->removalKey, &finger, &temp, 1);
cs->versionHistory.removeBefore(cs->oldestVersion, finger, combinedWriteConflictRanges.size() * 3 + 10);
cs->removalKey = finger.getValue();
}
g_removeBefore += timer() - t;
}
void ConflictBatch::checkReadConflictRanges() {
if (combinedReadConflictRanges.empty())
return;
cs->versionHistory.detectConflicts(
&combinedReadConflictRanges[0], combinedReadConflictRanges.size(), transactionConflictStatus);
}
void ConflictBatch::addConflictRanges(Version now,
std::vector<std::pair<StringRef, StringRef>>::iterator begin,
std::vector<std::pair<StringRef, StringRef>>::iterator end,
SkipList* part) {
const int count = end - begin;
static_assert(sizeof(*begin) == sizeof(StringRef) * 2,
"Write Conflict Range type not convertible to two StringPtrs");
const StringRef* strings = reinterpret_cast<const StringRef*>(&*begin);
const int stringCount = count * 2;
const int stripeSize = 16;
SkipList::Finger fingers[stripeSize];
int temp[stripeSize];
int stripes = (stringCount + stripeSize - 1) / stripeSize;
int ss = stringCount - (stripes - 1) * stripeSize;
for (int s = stripes - 1; s >= 0; s--) {
part->find(&strings[static_cast<size_t>(s) * stripeSize], fingers, temp, ss);
part->addConflictRanges(fingers, ss / 2, now);
ss = stripeSize;
}
}
void ConflictBatch::mergeWriteConflictRanges(Version now) {
if (combinedWriteConflictRanges.empty())
return;
addConflictRanges(now, combinedWriteConflictRanges.begin(), combinedWriteConflictRanges.end(), &cs->versionHistory);
}
void ConflictBatch::combineWriteConflictRanges() {
int activeWriteCount = 0;
for (const KeyInfo& point : points) {
if (point.write && !transactionConflictStatus[point.transaction]) {
if (point.begin) {
activeWriteCount++;
if (activeWriteCount == 1)
combinedWriteConflictRanges.emplace_back(point.key, KeyRef());
} else /*if (point.end)*/ {
activeWriteCount--;
if (activeWriteCount == 0)
combinedWriteConflictRanges.back().second = point.key;
}
}
}
}
namespace {
StringRef setK(Arena& arena, int i) {
char t[sizeof(i)];
*(int*)t = i;
const int keySize = 16;
char* ss = new (arena) char[keySize];
for (int c = 0; c < keySize - sizeof(i); c++)
ss[c] = '.';
for (int c = 0; c < sizeof(i); c++)
ss[c + keySize - sizeof(i)] = t[sizeof(i) - 1 - c];
return StringRef((const uint8_t*)ss, keySize);
}
void miniConflictSetTest() {
for (int i = 0; i < 2000000; i++) {
int size = 64 * 5; // Also run 64*64*5 to test multiple words of andValues and orValues
MiniConflictSet mini(size);
for (int j = 0; j < 2; j++) {
int a = deterministicRandom()->randomInt(0, size);
int b = deterministicRandom()->randomInt(a, size);
mini.set(a, b);
}
for (int j = 0; j < 4; j++) {
int a = deterministicRandom()->randomInt(0, size);
int b = deterministicRandom()->randomInt(a, size);
mini.any(a, b); // Tests correctness internally
}
}
printf("miniConflictSetTest complete\n");
}
void operatorLessThanTest() {
{ // Longer strings before shorter strings.
KeyInfo a("hello"_sr, /*begin=*/false, /*write=*/true, 0, nullptr);
KeyInfo b("hello\0"_sr, /*begin=*/false, /*write=*/false, 0, nullptr);
ASSERT(a < b);
ASSERT(!(b < a));
ASSERT(!(a == b));
}
{ // Reads before writes.
KeyInfo a("hello"_sr, /*begin=*/false, /*write=*/false, 0, nullptr);
KeyInfo b("hello"_sr, /*begin=*/false, /*write=*/true, 0, nullptr);
ASSERT(a < b);
ASSERT(!(b < a));
ASSERT(!(a == b));
}
{ // Begin reads after writes.
KeyInfo a("hello"_sr, /*begin=*/false, /*write=*/true, 0, nullptr);
KeyInfo b("hello"_sr, /*begin=*/true, /*write=*/false, 0, nullptr);
ASSERT(a < b);
ASSERT(!(b < a));
ASSERT(!(a == b));
}
{ // Begin writes after writes.
KeyInfo a("hello"_sr, /*begin=*/false, /*write=*/true, 0, nullptr);
KeyInfo b("hello"_sr, /*begin=*/true, /*write=*/true, 0, nullptr);
ASSERT(a < b);
ASSERT(!(b < a));
ASSERT(!(a == b));
}
}
} // namespace
void skipListTest() {
printf("Skip list test\n");
miniConflictSetTest();
operatorLessThanTest();
setAffinity(0);
double start;
ConflictSet* cs = newConflictSet();
Arena testDataArena;
VectorRef<VectorRef<KeyRangeRef>> testData;
const int batches = 500; // deterministicRandom()->randomInt(500, 5000);
const int data_per_batch = 5000;
testData.resize(testDataArena, batches);
std::vector<std::vector<uint8_t>> success(batches);
std::vector<std::vector<uint8_t>> success2(batches);
for (int i = 0; i < batches; i++) {
testData[i].resize(testDataArena, data_per_batch);
success[i].assign(data_per_batch, false);
success2[i].assign(data_per_batch, false);
for (int j = 0; j < data_per_batch; j++) {
int key = deterministicRandom()->randomInt(0, 20000000);
int key2 = key + 1 + deterministicRandom()->randomInt(0, 10);
testData[i][j] = KeyRangeRef(setK(testDataArena, key), setK(testDataArena, key2));
}
}
printf("Test data generated: %d batches, %d/batch\n", batches, data_per_batch);
printf("Running\n");
int readCount = 1, writeCount = 1;
int cranges = 0, tcount = 0;
start = timer();
std::vector<std::vector<int>> nonConflict(batches);
Version version = 0;
for (const auto& data : testData) {
Arena buf;
std::vector<CommitTransactionRef> trs;
double t = timer();
for (int j = 0; j + readCount + writeCount <= data.size(); j += readCount + writeCount) {
CommitTransactionRef tr;
for (int k = 0; k < readCount; k++) {
KeyRangeRef r(buf, data[j + k]);
tr.read_conflict_ranges.push_back(buf, r);
}
for (int k = 0; k < writeCount; k++) {
KeyRangeRef r(buf, data[j + readCount + k]);
tr.write_conflict_ranges.push_back(buf, r);
}
cranges += tr.read_conflict_ranges.size() + tr.write_conflict_ranges.size();
tr.read_snapshot = version;
trs.push_back(tr);
}
tcount += trs.size();
g_buildTest += timer() - t;
t = timer();
ConflictBatch batch(cs);
for (const auto& tr : trs) {
batch.addTransaction(tr, version);
}
g_add += timer() - t;
t = timer();
batch.detectConflicts(version + 50, version, nonConflict[version]);
g_detectConflicts += timer() - t;
version++;
}
double elapsed = timer() - start;
printf("New conflict set: %0.3f sec\n", elapsed);
printf(" %0.3f Mtransactions/sec\n", tcount / elapsed / 1e6);
printf(" %0.3f Mkeys/sec\n", cranges * 2 / elapsed / 1e6);
elapsed = g_detectConflicts.getValue();
printf("Detect only: %0.3f sec\n", elapsed);
printf(" %0.3f Mtransactions/sec\n", tcount / elapsed / 1e6);
printf(" %0.3f Mkeys/sec\n", cranges * 2 / elapsed / 1e6);
elapsed = g_checkRead.getValue() + g_merge.getValue();
printf("Skiplist only: %0.3f sec\n", elapsed);
printf(" %0.3f Mtransactions/sec\n", tcount / elapsed / 1e6);
printf(" %0.3f Mkeys/sec\n", cranges * 2 / elapsed / 1e6);
printf("Performance counters:\n");
for (const auto& counter : skc) {
printf("%20s: %s\n", counter->getMetric().name().c_str(), counter->getMetric().formatted().c_str());
}
printf("%d entries in version history\n", cs->versionHistory.count());
}
TEST_CASE("/fdbserver/skiplist/miniConflictSetCompatibility") {
// Unit test written by Claude AI assistant
// Reference implementation using std::vector<bool> for comparison
class ReferenceMiniConflictSet {
std::vector<bool> bits;
public:
explicit ReferenceMiniConflictSet(size_t size) : bits(size, false) {}
void set(int begin, int end) {
if (end <= begin)
return;
for (int i = begin; i < end; i++) {
bits[i] = true;
}
}
bool any(int begin, int end) const {
if (end <= begin)
return false;
for (int i = begin; i < end; i++) {
if (bits[i])
return true;
}
return false;
}
void clear() { std::fill(bits.begin(), bits.end(), false); }
};
auto testCase = [](size_t size,
const std::vector<std::pair<int, int>>& setOperations,
const std::vector<std::pair<int, int>>& anyQueries) {
MiniConflictSet mcs(size);
ReferenceMiniConflictSet ref(size);
// Apply set operations to both implementations
for (const auto& op : setOperations) {
mcs.set(op.first, op.second);
ref.set(op.first, op.second);
}
// Compare any() results
for (const auto& query : anyQueries) {
bool mcsResult = mcs.any(query.first, query.second);
bool refResult = ref.any(query.first, query.second);
ASSERT(mcsResult == refResult);
}
};
// Test 1: Edge cases - empty ranges and boundary conditions
// Rationale: Empty ranges (begin == end) should be no-ops but could cause issues
// if the implementation doesn't handle them properly. This catches off-by-one errors.
{
std::vector<std::pair<int, int>> setOps = { { 0, 0 }, { 5, 5 }, { 10, 10 } }; // Empty ranges
std::vector<std::pair<int, int>> queries = { { 0, 1 }, { 5, 6 }, { 9, 11 }, { 0, 64 } };
testCase(64, setOps, queries);
}
// Test 2: Single bit operations at critical positions
// Rationale: Single bits test the minimal unit of operation and catch issues with
// bit indexing, especially at word boundaries (0, 63) and middle positions.
{
std::vector<std::pair<int, int>> setOps = { { 0, 1 }, { 63, 64 }, { 32, 33 } };
std::vector<std::pair<int, int>> queries = { { 0, 1 }, { 63, 64 }, { 32, 33 }, { 0, 64 }, { 31, 34 } };
testCase(64, setOps, queries);
}
// Test 3: Full word operations (64 bits)
// Rationale: Setting/querying entire 64-bit words tests the numBits == 64 special case
// in set() method and ensures correct mask generation for full words.
{
std::vector<std::pair<int, int>> setOps = { { 0, 64 } };
std::vector<std::pair<int, int>> queries = { { 0, 64 }, { 0, 32 }, { 32, 64 }, { 10, 50 } };
testCase(64, setOps, queries);
}
// Test 4: Multi-word operations
// Rationale: Operations spanning multiple 64-bit words exercise the multi-word logic
// in both set() and any(). This catches issues with word iteration and mask application.
{
std::vector<std::pair<int, int>> setOps = { { 0, 128 }, { 64, 192 }, { 100, 300 } };
std::vector<std::pair<int, int>> queries = { { 0, 64 }, { 64, 128 }, { 128, 192 }, { 0, 320 }, { 50, 150 } };
testCase(320, setOps, queries);
}
// Test 5: Overlapping ranges
// Rationale: Overlapping set operations test that bits remain set correctly when
// multiple ranges affect the same positions. This ensures proper OR semantics.
{
std::vector<std::pair<int, int>> setOps = { { 10, 50 }, { 30, 70 }, { 60, 100 } };
std::vector<std::pair<int, int>> queries = { { 0, 10 }, { 10, 30 }, { 30, 50 }, { 50, 60 },
{ 60, 70 }, { 70, 100 }, { 0, 128 } };
testCase(128, setOps, queries);
}
// Test 6: Word boundary edge cases
// Rationale: Operations that cross 64-bit word boundaries are prone to off-by-one
// errors and incorrect mask calculations. This specifically targets boundary crossings.
{
std::vector<std::pair<int, int>> setOps = { { 63, 65 }, { 127, 129 }, { 191, 193 } };
std::vector<std::pair<int, int>> queries = { { 62, 64 }, { 63, 64 }, { 64, 65 },
{ 126, 128 }, { 127, 128 }, { 128, 129 } };
testCase(256, setOps, queries);
}
// Test 7: Random stress test with small size
// Rationale: Random testing catches edge cases that structured tests might miss.
// Small size ensures good coverage of single and double-word scenarios.
{
std::vector<std::pair<int, int>> setOps, queries;
for (int i = 0; i < 20; i++) {
int a = deterministicRandom()->randomInt(0, 128);
int b = deterministicRandom()->randomInt(a, 128);
setOps.push_back({ a, b });
}
for (int i = 0; i < 50; i++) {
int a = deterministicRandom()->randomInt(0, 128);
int b = deterministicRandom()->randomInt(a, 128);
queries.push_back({ a, b });
}
testCase(128, setOps, queries);
}
// Test 8: Random stress test with large size
// Rationale: Larger random testing stresses multi-word logic and can expose
// performance issues or correctness bugs that only appear with many words.
{
std::vector<std::pair<int, int>> setOps, queries;
for (int i = 0; i < 50; i++) {
int a = deterministicRandom()->randomInt(0, 1000);
int b = deterministicRandom()->randomInt(a, 1000);
setOps.push_back({ a, b });
}
for (int i = 0; i < 100; i++) {
int a = deterministicRandom()->randomInt(0, 1000);
int b = deterministicRandom()->randomInt(a, 1000);
queries.push_back({ a, b });
}
testCase(1000, setOps, queries);
}
// Test 9: Large ranges spanning multiple words
// Rationale: Very large ranges test the efficiency and correctness of the multi-word
// implementation, ensuring it handles bulk operations without errors.
{
std::vector<std::pair<int, int>> setOps = { { 50, 500 }, { 200, 800 }, { 700, 900 } };
std::vector<std::pair<int, int>> queries = { { 0, 50 }, { 50, 200 }, { 200, 500 }, { 500, 700 },
{ 700, 800 }, { 800, 900 }, { 900, 1000 } };
testCase(1000, setOps, queries);
}
// Test 10: Clear operation test
// Rationale: The clear() operation must reset all bits to zero. This verifies
// that clear() works correctly and doesn't leave stray bits set.
{
MiniConflictSet mcs(64);
ReferenceMiniConflictSet ref(64);
// Set some bits
mcs.set(10, 50);
ref.set(10, 50);
// Verify they're set
ASSERT(mcs.any(20, 30) == ref.any(20, 30));
ASSERT(mcs.any(20, 30) == true);
// Clear and verify
mcs.clear();
ref.clear();
ASSERT(mcs.any(20, 30) == ref.any(20, 30));
ASSERT(mcs.any(20, 30) == false);
ASSERT(mcs.any(0, 64) == ref.any(0, 64));
ASSERT(mcs.any(0, 64) == false);
}
// Test 11: Patterns that might expose bit manipulation bugs
// Rationale: Alternating bit patterns create specific mask patterns that could
// expose bugs in bit manipulation logic, especially with carry/overflow issues.
{
// Alternating bits pattern
std::vector<std::pair<int, int>> setOps;
for (int i = 0; i < 128; i += 2) {
setOps.push_back({ i, i + 1 });
}
std::vector<std::pair<int, int>> queries = { { 0, 128 }, { 0, 64 }, { 64, 128 }, { 1, 127 } };
testCase(128, setOps, queries);
}
// Test 12: Verify specific bit positions in multi-word scenarios
// Rationale: First and last bits of each word are most prone to indexing errors.
// This test specifically targets these critical positions across multiple words.
{
std::vector<std::pair<int, int>> setOps = { { 0, 1 }, { 63, 64 }, { 64, 65 }, { 127, 128 } };
std::vector<std::pair<int, int>> queries = { { 0, 1 }, { 63, 64 }, { 64, 65 }, { 127, 128 }, { 0, 128 } };
testCase(128, setOps, queries);
}
// Test 13: Comprehensive word boundary testing
// Rationale: Word boundaries (multiples of 64) are critical because bit manipulation
// logic often has special cases when ranges cross 64-bit word boundaries.
// This test ensures set() and any() work correctly when ranges start/end at or near
// word boundaries, catching off-by-one errors and incorrect mask calculations.
{
const int testSize = 320; // 5 words
std::vector<std::pair<int, int>> setOps, queries;
// Test every word boundary systematically
for (int wordBoundary : { 64, 128, 192, 256 }) {
for (int offset : { -1, 0, 1 }) {
int pos = wordBoundary + offset;
if (pos >= 0 && pos < testSize) {
// Single bit at boundary
setOps.push_back({ pos, pos + 1 });
queries.push_back({ pos, pos + 1 });
// Range crossing boundary
if (pos > 5 && pos < testSize - 5) {
setOps.push_back({ pos - 2, pos + 3 });
queries.push_back({ pos - 2, pos + 3 });
queries.push_back({ pos - 1, pos + 2 });
}
}
}
}
// Test ranges that span exactly 1, 2, 3+ words
setOps.push_back({ 0, 64 }); // Exactly 1 word
setOps.push_back({ 64, 192 }); // Exactly 2 words
setOps.push_back({ 128, 320 }); // Exactly 3 words
queries.push_back({ 0, 64 });
queries.push_back({ 64, 192 });
queries.push_back({ 128, 320 });
queries.push_back({ 0, 320 });
testCase(testSize, setOps, queries);
}
// Test 14: Mask generation edge cases
// Rationale: The set() method has special logic for numBits == 64 (lines 905-909)
// and different code paths for single vs multi-word operations. This test specifically
// targets those branches to ensure correct mask generation in all scenarios.
{
std::vector<std::pair<int, int>> setOps, queries;
// Test numBits == 64 case (single word, full mask)
setOps.push_back({ 0, 64 });
setOps.push_back({ 64, 128 });
queries.push_back({ 0, 64 });
queries.push_back({ 64, 128 });
// Test begin % 64 == 0 and end % 64 == 0 cases
setOps.push_back({ 128, 192 });
setOps.push_back({ 192, 256 });
queries.push_back({ 128, 192 });
queries.push_back({ 192, 256 });
// Test single-word ranges with different start positions
for (int start = 0; start < 64; start += 7) {
setOps.push_back({ start, start + 1 });
setOps.push_back({ start + 64, start + 65 });
queries.push_back({ start, start + 1 });
queries.push_back({ start + 64, start + 65 });
}
testCase(256, setOps, queries);
}
// Test 15: Extreme size testing
// Rationale: Large sizes (10,000+ bits) stress-test the multi-word logic and can expose
// issues that only appear when dealing with many 64-bit words. This catches performance
// issues and ensures correctness scales beyond typical use cases.
{
const int largeSize = 10000;
std::vector<std::pair<int, int>> setOps, queries;
// Large ranges across many words
setOps.push_back({ 100, 5000 });
setOps.push_back({ 3000, 8000 });
setOps.push_back({ 7000, 9500 });
// Query various sections
queries.push_back({ 0, 100 });
queries.push_back({ 100, 3000 });
queries.push_back({ 3000, 5000 });
queries.push_back({ 5000, 7000 });
queries.push_back({ 7000, 8000 });
queries.push_back({ 8000, 9500 });
queries.push_back({ 9500, largeSize });
queries.push_back({ 0, largeSize });
testCase(largeSize, setOps, queries);
}
return Void();
}
Reference in New Issue View Git Blame Copy Permalink